{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:08.713927016Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:54435"} {"msg":"region 0e616c54-e923-4dd7-8297-bb8a8acc8682 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:08.714020127Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:08.714705531Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"uri":"/crucible/0/regions","method":"POST","req_id":"8f0c6890-b0f0-4737-a14b-d48510c1f4e6","remote_addr":"[fd00:1122:3344:101::b]:54435","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":720,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:08.714726751Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_28fb7d98-1494-4570-b25a-aa56857fbe19/crucible/regions/0e616c54-e923-4dd7-8297-bb8a8acc8682","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:08.726577848Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_28fb7d98-1494-4570-b25a-aa56857fbe19/crucible/regions/0e616c54-e923-4dd7-8297-bb8a8acc8682","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:08.726611179Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"0e616c54-e923-4dd7-8297-bb8a8acc8682\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/0e616c54-e923-4dd7-8297-bb8a8acc8682\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:08.803499706Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"region":"0e616c54-e923-4dd7-8297-bb8a8acc8682","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"0e616c54-e923-4dd7-8297-bb8a8acc8682\" \"--data\" \"/data/regions/0e616c54-e923-4dd7-8297-bb8a8acc8682\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:08.805466403Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"region":"0e616c54-e923-4dd7-8297-bb8a8acc8682","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:08.935296187Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"region":"0e616c54-e923-4dd7-8297-bb8a8acc8682","component":"worker"} {"msg":"region 0e616c54-e923-4dd7-8297-bb8a8acc8682 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:08.935355808Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:08.935511641Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-0e616c54-e923-4dd7-8297-bb8a8acc8682","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:08.937611632Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-0e616c54-e923-4dd7-8297-bb8a8acc8682","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:08.946840019Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:08.964972597Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:08.973843927Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"uri":"/crucible/0/regions","method":"POST","req_id":"bce4ef55-fb32-4c3b-ba7f-327530ec5cbc","remote_addr":"[fd00:1122:3344:101::b]:54435","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":107,"response_code":"200"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-0e616c54-e923-4dd7-8297-bb8a8acc8682","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:08.974129573Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/0e616c54-e923-4dd7-8297-bb8a8acc8682","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:08.97453233Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:08.974613382Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:08.974712194Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:08.974804686Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:08.987651782Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:08.999083842Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"component":"worker"} {"msg":"region b2acd9d8-3b2a-4126-93ad-b84a9cd12627 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:15.223768966Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:15.22395308Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"uri":"/crucible/0/regions","method":"POST","req_id":"6d55db54-aaf8-4778-a2cf-b010ba2e5a46","remote_addr":"[fd00:1122:3344:101::b]:54435","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":232,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:15.22396931Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_28fb7d98-1494-4570-b25a-aa56857fbe19/crucible/regions/b2acd9d8-3b2a-4126-93ad-b84a9cd12627","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:15.233595225Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_28fb7d98-1494-4570-b25a-aa56857fbe19/crucible/regions/b2acd9d8-3b2a-4126-93ad-b84a9cd12627","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:15.233620646Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"b2acd9d8-3b2a-4126-93ad-b84a9cd12627\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/b2acd9d8-3b2a-4126-93ad-b84a9cd12627\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:15.298219396Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"region":"b2acd9d8-3b2a-4126-93ad-b84a9cd12627","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"b2acd9d8-3b2a-4126-93ad-b84a9cd12627\" \"--data\" \"/data/regions/b2acd9d8-3b2a-4126-93ad-b84a9cd12627\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:15.298274407Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"region":"b2acd9d8-3b2a-4126-93ad-b84a9cd12627","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:15.39267266Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"uri":"/crucible/0/regions","method":"POST","req_id":"c7253f38-ef5f-4572-a1fd-63e68a9d2626","remote_addr":"[fd00:1122:3344:101::b]:54435","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":98,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:15.435107605Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"region":"b2acd9d8-3b2a-4126-93ad-b84a9cd12627","component":"worker"} {"msg":"region b2acd9d8-3b2a-4126-93ad-b84a9cd12627 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:15.435135876Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:15.435262518Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-b2acd9d8-3b2a-4126-93ad-b84a9cd12627","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:15.437210576Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b2acd9d8-3b2a-4126-93ad-b84a9cd12627","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:15.444151019Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:15.459393942Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b2acd9d8-3b2a-4126-93ad-b84a9cd12627","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:15.468330863Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b2acd9d8-3b2a-4126-93ad-b84a9cd12627","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:15.468606209Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:15.46866267Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:15.468711891Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:15.468754091Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:15.47803016Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:15.489562781Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:15.936240149Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"uri":"/crucible/0/regions","method":"POST","req_id":"edca7040-a411-458b-9c31-cd9f94277c05","remote_addr":"[fd00:1122:3344:101::b]:54435","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":67,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:43.260709041Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"uri":"/crucible/0/regions/b2acd9d8-3b2a-4126-93ad-b84a9cd12627","method":"GET","req_id":"ffcc9f0a-3be3-4d3f-954f-90aa49eb0b24","remote_addr":"[fd00:1122:3344:101::b]:54435","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":55,"response_code":"200"} {"msg":"path is \"/data/regions/b2acd9d8-3b2a-4126-93ad-b84a9cd12627\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:43.26481976Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"component":"datafile"} {"msg":"dataset is oxp_28fb7d98-1494-4570-b25a-aa56857fbe19/crucible/regions/b2acd9d8-3b2a-4126-93ad-b84a9cd12627","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:43.276600486Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"component":"datafile"} {"msg":"region b2acd9d8-3b2a-4126-93ad-b84a9cd12627 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:43.286755601Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:43.286943015Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:43.287014266Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"uri":"/crucible/0/regions/b2acd9d8-3b2a-4126-93ad-b84a9cd12627","method":"DELETE","req_id":"9ec98336-36ec-486b-8338-1d9c1b657a23","remote_addr":"[fd00:1122:3344:101::b]:54435","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":22863,"response_code":"204"} {"msg":"disabling downstairs instance: downstairs-b2acd9d8-3b2a-4126-93ad-b84a9cd12627 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:43.288907903Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:43.290917961Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"uri":"/crucible/0/regions/b2acd9d8-3b2a-4126-93ad-b84a9cd12627","method":"GET","req_id":"bd3b6fe0-81e3-4f4e-916f-eb04bdacd3cc","remote_addr":"[fd00:1122:3344:101::b]:54435","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":30,"response_code":"200"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:43.295119412Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_28fb7d98-1494-4570-b25a-aa56857fbe19/crucible/regions/b2acd9d8-3b2a-4126-93ad-b84a9cd12627\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:43.305622094Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"region":"b2acd9d8-3b2a-4126-93ad-b84a9cd12627","component":"worker"} {"msg":"region b2acd9d8-3b2a-4126-93ad-b84a9cd12627 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:43.354760207Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:43.372164471Z","hostname":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","pid":23412,"uri":"/crucible/0/regions/b2acd9d8-3b2a-4126-93ad-b84a9cd12627","method":"GET","req_id":"c86a151c-6408-4237-be2b-07b6b2d461c1","remote_addr":"[fd00:1122:3344:101::b]:54435","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":686,"response_code":"200"}