Upgrade from 4.2.4 to 5.x failed

we are using the concourse-bosh-deployment and have had no issues going from 4.2.1 up through 4.2.4, but when we try to go to any 5.x releases we get this error message.

    Using environment '10.100.20.31' as client 'ops_manager'
    
    Using deployment 'concourse'

    Evaluating manifest:
      Expected to find exactly one matching array item for path 
    **'/instance_groups/name=web/jobs/name=atc'** but found 0
    
    Exit code 1

I’ve looked over the current deployment manifest and everything looks good to me. Anyone come across this before?

this is the currently deployed manifest for 4.2.4

instance_groups:
- azs:
  - az1
  instances: 1
  jobs:
  - name: atc
    properties:
      add_local_users:
      - admin:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
      bind_port: 80
      credhub:
        client_id: concourse_to_credhub
        client_secret: ((concourse_to_credhub_secret))
        tls:
          ca_cert:
            certificate: ((atc_tls.ca))
        url: https://concourse.domain.name:8844
      external_url: https://concourse.domain.name
      ldap_auth:
        bind_dn: CN=ldap,OU=Accts,DC=domain,DC=name
        bind_pw: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
        group_search_base_dn: OU=Groups,DC=domain,DC=name
        group_search_filter: memberOf=CN=Admins,OU=Groups,DC=domain,DC=name
        host: ldap.domain.name
        insecure_no_ssl: true
        insecure_skip_verify: true
        start_tls: false
        user_search_base_dn: OU=Users,DC=domain,DC=name
        user_search_email_attr: mail
        user_search_filter: (objectClass=person)
        user_search_id_attr: uid
        user_search_name_attr: uid
        user_search_scope: sub
        user_search_username: uid
      main_team:
        auth:
          local:
            users:
            - admin
      postgresql:
        database: atc
        role:
          name: concourse
          password: ((postgres_password))
      prometheus:
        bind_ip: 0.0.0.0
        bind_port: 9391
      tls_bind_port: 443
      tls_cert: ((atc_tls.certificate))
      tls_key: ((atc_tls.private_key))
      token_signing_key: ((token_signing_key))
    release: concourse
  - name: tsa
    properties:
      authorized_keys:
      - ((worker_key.public_key))
      host_key: ((tsa_host_key))
      token_signing_key: ((token_signing_key))
    release: concourse
  - name: uaa
    properties:
      encryption:
        active_key_label: KEY-1
        encryption_keys:
        - label: KEY-1
          passphrase: ((encryption-keys-passphrase))
      login:
        saml:
          serviceProviderCertificate: ((atc_tls.certificate))
          serviceProviderKey: ((atc_tls.private_key))
          serviceProviderKeyPassword: ""
      uaa:
        admin:
          client_secret: ((uaa-admin))
        clients:
          concourse_to_credhub:
            access-token-validity: 1200
            authorities: credhub.read,credhub.write
            authorized-grant-types: client_credentials
            override: true
            refresh-token-validity: 3600
            scope: ""
            secret: ((concourse_to_credhub_secret))
          credhub_cli:
            access-token-validity: 1200
            authorities: uaa.resource
            authorized-grant-types: password,refresh_token
            override: true
            refresh-token-validity: 3600
            scope: credhub.read,credhub.write
            secret: ""
        jwt:
          policy:
            active_key_id: key-1
            keys:
              key-1:
                signingKey: ((uaa-jwt.private_key))
          revocable: true
        logging_level: INFO
        login:
          client_secret: ((uaa-login))
        port: 8181
        scim:
          users:
          - groups:
            - scim.write
            - scim.read
            - bosh.admin
            - credhub.read
            - credhub.write
            name: admin
            password: ((uaa-users-admin))
        sslCertificate: ((atc_tls.certificate))
        sslPrivateKey: ((atc_tls.private_key))
        url: https://concourse.host.name:8443
        zones:
          internal:
            hostnames: []
      uaadb:
        databases:
        - name: uaa
          tag: uaa
        db_scheme: postgresql
        port: 5432
        roles:
        - name: uaa
          password: ((uaa-db-password))
          tag: admin
    release: uaa
  - name: credhub
    properties:
      credhub:
        authentication:
          uaa:
            ca_certs:
            - ((atc_tls.ca))
            url: https://concourse.host.name:8443
            verification_key: ((uaa-jwt.public_key))
        data_storage:
          database: credhub
          password: ((credhub-db-password))
          port: 5432
          require_tls: false
          type: postgres
          username: credhub
        encryption:
          keys:
          - active: true
            encryption_password: ((credhub-encryption-password))
            provider_name: int
          providers:
          - name: int
            type: internal
        log_level: debug
        port: 8844
        tls: ((atc_tls))
    release: credhub
  name: web
  networks:
  - name: concourse
    static_ips:
    - 10.100.20.45
  stemcell: xenial
  vm_type: large
- azs:
  - az1
  instances: 1
  jobs:
  - name: postgres
    properties:
      databases:
        databases:
        - name: atc
        - name: credhub
        - name: uaa
        port: 5432
        roles:
        - name: concourse
          password: ((postgres_password))
        - name: credhub
          password: ((credhub-db-password))
        - name: uaa
          password: ((uaa-db-password))
    release: postgres
  - name: bbr-atcdb
    properties:
      postgresql:
        database: atc
        role:
          name: concourse
          password: ((postgres_password))
    release: concourse
  - name: database-backup-restorer
    release: backup-and-restore-sdk
  name: db
  networks:
  - name: concourse
  persistent_disk_type: 100GB
  stemcell: xenial
  vm_type: large
- azs:
  - az1
  instances: 10
  jobs:
  - consumes:
      baggageclaim:
        from: worker-baggageclaim
    name: worker
    properties:
      drain_timeout: 10m
      tsa:
        worker_key: ((worker_key))
    release: concourse
  - name: baggageclaim
    provides:
      baggageclaim:
        as: worker-baggageclaim
    release: concourse
  - name: garden
    properties:
      garden:
        listen_address: 0.0.0.0:7777
        listen_network: tcp
    release: garden-runc
  name: worker
  networks:
  - name: concourse
  stemcell: xenial
  vm_type: concourse.worker
name: concourse
releases:
- name: concourse
  sha1: 56122002ce49af09b03b8c3a49bdb7578ee2868a
  url: https://bosh.io/d/github.com/concourse/concourse-bosh-release?v=4.2.4
  version: 4.2.4
- name: garden-runc
  sha1: 54cbb89cae1be0708aa056185671665d7f4b2a4f
  url: https://bosh.io/d/github.com/cloudfoundry/garden-runc-release?v=1.13.1
  version: 1.13.1
- name: postgres
  sha1: 3dd10b417b21cfa3257f1cc891e9e46f02fefe16
  url: https://bosh.io/d/github.com/cloudfoundry/postgres-release?v=36
  version: "36"
- name: backup-and-restore-sdk
  version: latest
- name: uaa
  sha1: a7c14357ae484e89e547f4f207fb8de36d2b0966
  url: https://bosh.io/d/github.com/cloudfoundry/uaa-release?v=60
  version: 60
- name: credhub
  sha1: 648658efdef2ff18a69914d958bcd7ebfa88027a
  url: https://bosh.io/d/github.com/pivotal-cf/credhub-release?v=1.9.3
  version: 1.9.3
stemcells:
- alias: xenial
  os: ubuntu-xenial
  version: latest
update:
  canaries: 1
  canary_watch_time: 30000-1200000
  max_in_flight: 3
  serial: false
  update_watch_time: 5000-1200000
variables:
- name: postgres_password
  type: password
- name: token_signing_key
  type: rsa
- name: tsa_host_key
  type: ssh
- name: worker_key
  type: ssh
- name: atc_ca
  options:
    common_name: atcCA
    is_ca: true
  type: certificate
- name: atc_tls
  options:
    alternative_names:
    - concourse.host.name
    ca: atc_ca
    organization: atcOrg
  type: certificate
- name: atc-db-password
  type: password
- name: credhub-encryption-password
  options:
    length: 40
  type: password
- name: credhub-db-password
  type: password
- name: uaa-jwt
  options:
    key_length: 4096
  type: rsa
- name: uaa-users-admin
  type: password
- name: uaa-admin
  type: password
- name: uaa-login
  type: password
- name: uaa-credhub-admin
  type: password
- name: uaa-db-admin
  type: password
- name: uaa-db-password
  type: password
- name: concourse_to_credhub_secret
  type: password
- name: credhub_cli_password
  type: password
- name: concourse_client_secret
  type: password
- name: encryption-keys-passphrase
  options:
    length: 40
  type: password

As of Concourse 5 there are no longer atc and tsa jobs in the bosh release. This is most likely your problem if you’re deploying with your 4.2.4 manifest without changing it.

You can see an example manifest for Concourse 5 in the concourse-bosh-deployment repo.