[ Jan 18 05:33:10 Enabled. ] [ Jan 18 05:33:10 Rereading configuration. ] [ Jan 18 05:33:15 Rereading configuration. ] [ Jan 18 05:33:21 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_16bb833e-d265-4e47-8c19-1cb9a8fa2175/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"dataset: \"oxp_16bb833e-d265-4e47-8c19-1cb9a8fa2175/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:33:21.961624939Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538} {"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:33:21.962574943Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:33:21.962583593Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:33:21.98251271Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:33:22.120878872Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.394602819Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:49368"} {"msg":"region c9a9186a-9d34-472c-9bbc-198747552ab1 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.420059469Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.425064633Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"uri":"/crucible/0/regions","method":"POST","req_id":"17adfd48-fe61-4eaa-b8cc-0ac69dad0086","remote_addr":"[fd00:1122:3344:101::b]:49368","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":13538,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.425082474Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_16bb833e-d265-4e47-8c19-1cb9a8fa2175/crucible/regions/c9a9186a-9d34-472c-9bbc-198747552ab1","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.434142879Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_16bb833e-d265-4e47-8c19-1cb9a8fa2175/crucible/regions/c9a9186a-9d34-472c-9bbc-198747552ab1","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.434165049Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"c9a9186a-9d34-472c-9bbc-198747552ab1\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/c9a9186a-9d34-472c-9bbc-198747552ab1\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.493912341Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"region":"c9a9186a-9d34-472c-9bbc-198747552ab1","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"c9a9186a-9d34-472c-9bbc-198747552ab1\" \"--data\" \"/data/regions/c9a9186a-9d34-472c-9bbc-198747552ab1\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.49588413Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"region":"c9a9186a-9d34-472c-9bbc-198747552ab1","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.638683021Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"uri":"/crucible/0/regions","method":"POST","req_id":"f923c444-5ce5-4777-bebd-4c4d794cd618","remote_addr":"[fd00:1122:3344:101::b]:49368","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":109,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.647892438Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"region":"c9a9186a-9d34-472c-9bbc-198747552ab1","component":"worker"} {"msg":"region c9a9186a-9d34-472c-9bbc-198747552ab1 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.647913619Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.64802562Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-c9a9186a-9d34-472c-9bbc-198747552ab1","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.648415716Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-c9a9186a-9d34-472c-9bbc-198747552ab1","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.652298434Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.665318938Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-c9a9186a-9d34-472c-9bbc-198747552ab1","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.675082664Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c9a9186a-9d34-472c-9bbc-198747552ab1","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.675664433Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.677574961Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.677642772Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.677687473Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.68216723Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.685807184Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:11.353959633Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"uri":"/crucible/0/regions","method":"POST","req_id":"f81d1750-d16e-46c2-b4f4-6262fa936749","remote_addr":"[fd00:1122:3344:101::b]:49368","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":73,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:07.714634109Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:48773"} {"msg":"region b0740894-8f8e-4b9f-977d-73ad3a41efaf state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:07.714773321Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:07.715421201Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:07.715432751Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"uri":"/crucible/0/regions","method":"POST","req_id":"7dd72778-a89b-40eb-b0d7-87a40a0817f5","remote_addr":"[fd00:1122:3344:101::b]:48773","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":697,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_16bb833e-d265-4e47-8c19-1cb9a8fa2175/crucible/regions/b0740894-8f8e-4b9f-977d-73ad3a41efaf","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:07.726436555Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_16bb833e-d265-4e47-8c19-1cb9a8fa2175/crucible/regions/b0740894-8f8e-4b9f-977d-73ad3a41efaf","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:07.726459066Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"b0740894-8f8e-4b9f-977d-73ad3a41efaf\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/b0740894-8f8e-4b9f-977d-73ad3a41efaf\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:07.779265203Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"region":"b0740894-8f8e-4b9f-977d-73ad3a41efaf","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"b0740894-8f8e-4b9f-977d-73ad3a41efaf\" \"--data\" \"/data/regions/b0740894-8f8e-4b9f-977d-73ad3a41efaf\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:07.779323135Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"region":"b0740894-8f8e-4b9f-977d-73ad3a41efaf","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:07.845780495Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"region":"b0740894-8f8e-4b9f-977d-73ad3a41efaf","component":"worker"} {"msg":"region b0740894-8f8e-4b9f-977d-73ad3a41efaf state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:07.845817406Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:07.845906667Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-b0740894-8f8e-4b9f-977d-73ad3a41efaf","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:07.846293183Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b0740894-8f8e-4b9f-977d-73ad3a41efaf","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:07.851637223Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:07.864948212Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b0740894-8f8e-4b9f-977d-73ad3a41efaf","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:07.868703288Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b0740894-8f8e-4b9f-977d-73ad3a41efaf","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:07.868969701Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:07.869029772Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:07.869073793Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:07.869114544Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:07.873112873Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"existing directory value /dev/null does not match /data/regions/c9a9186a-9d34-472c-9bbc-198747552ab1","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:07.877677542Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:07.877760033Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:07.877840964Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:07.877848244Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:07.877888614Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-c9a9186a-9d34-472c-9bbc-198747552ab1","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:07.877934865Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c9a9186a-9d34-472c-9bbc-198747552ab1","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:07.878010726Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:07.878065437Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:07.878116688Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:07.878162719Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:07.889142942Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:07.889590979Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:07.911949586Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"uri":"/crucible/0/regions","method":"POST","req_id":"7c089e3e-e6af-4cb0-baa7-f0b7c3363d3f","remote_addr":"[fd00:1122:3344:101::b]:48773","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":92,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:08.813772349Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"uri":"/crucible/0/regions/c9a9186a-9d34-472c-9bbc-198747552ab1","method":"GET","req_id":"e6db38a4-ea58-4b15-b99e-64635343c6b2","remote_addr":"[fd00:1122:3344:101::b]:48773","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":72,"response_code":"200"} {"msg":"path is \"/data/regions/c9a9186a-9d34-472c-9bbc-198747552ab1\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:08.81984156Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"datafile"} {"msg":"dataset is oxp_16bb833e-d265-4e47-8c19-1cb9a8fa2175/crucible/regions/c9a9186a-9d34-472c-9bbc-198747552ab1","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:08.828795963Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"datafile"} {"msg":"snapshot is oxp_16bb833e-d265-4e47-8c19-1cb9a8fa2175/crucible/regions/c9a9186a-9d34-472c-9bbc-198747552ab1@55708258-8abd-4c4b-8508-8aec30cbe904","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:08.837575654Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_16bb833e-d265-4e47-8c19-1cb9a8fa2175/crucible/regions/c9a9186a-9d34-472c-9bbc-198747552ab1\", \"55708258-8abd-4c4b-8508-8aec30cbe904\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:08.837603034Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:08.848168022Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"uri":"/crucible/0/regions/c9a9186a-9d34-472c-9bbc-198747552ab1/snapshots/55708258-8abd-4c4b-8508-8aec30cbe904","method":"GET","req_id":"e70322cc-31ff-471a-bdbe-e9f5fe690f7c","remote_addr":"[fd00:1122:3344:101::b]:48773","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":28920,"response_code":"200"} {"msg":"path is \"/data/regions/c9a9186a-9d34-472c-9bbc-198747552ab1\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:08.851106416Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"datafile"} {"msg":"dataset is oxp_16bb833e-d265-4e47-8c19-1cb9a8fa2175/crucible/regions/c9a9186a-9d34-472c-9bbc-198747552ab1","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:08.859121286Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"datafile"} {"msg":"snapshot is oxp_16bb833e-d265-4e47-8c19-1cb9a8fa2175/crucible/regions/c9a9186a-9d34-472c-9bbc-198747552ab1@55708258-8abd-4c4b-8508-8aec30cbe904","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:08.867042714Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_16bb833e-d265-4e47-8c19-1cb9a8fa2175/crucible/regions/c9a9186a-9d34-472c-9bbc-198747552ab1\", \"55708258-8abd-4c4b-8508-8aec30cbe904\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:08.867071494Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot c9a9186a-9d34-472c-9bbc-198747552ab1-55708258-8abd-4c4b-8508-8aec30cbe904 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:08.876029328Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"datafile"} {"msg":"applying SMF actions for region c9a9186a-9d34-472c-9bbc-198747552ab1 running snapshot 55708258-8abd-4c4b-8508-8aec30cbe904 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:08.876220791Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:08.876246041Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"uri":"/crucible/0/regions/c9a9186a-9d34-472c-9bbc-198747552ab1/snapshots/55708258-8abd-4c4b-8508-8aec30cbe904/run","method":"POST","req_id":"2aca71cf-770b-4679-9228-2688000b1fc9","remote_addr":"[fd00:1122:3344:101::b]:48773","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":25173,"response_code":"200"} {"msg":"existing directory value /dev/null does not match /data/regions/c9a9186a-9d34-472c-9bbc-198747552ab1","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:08.877783244Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:08.877863385Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:08.877941596Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:08.877948067Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:08.877985987Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-c9a9186a-9d34-472c-9bbc-198747552ab1","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:08.878028268Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c9a9186a-9d34-472c-9bbc-198747552ab1","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:08.878325632Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:08.878386043Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:08.878431754Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:08.878475034Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:08.881951466Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"creating missing snapshot instance snapshot-c9a9186a-9d34-472c-9bbc-198747552ab1-55708258-8abd-4c4b-8508-8aec30cbe904","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:08.882457274Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-c9a9186a-9d34-472c-9bbc-198747552ab1-55708258-8abd-4c4b-8508-8aec30cbe904","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:08.888678307Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-c9a9186a-9d34-472c-9bbc-198747552ab1-55708258-8abd-4c4b-8508-8aec30cbe904","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:08.897923914Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c9a9186a-9d34-472c-9bbc-198747552ab1/.zfs/snapshot/55708258-8abd-4c4b-8508-8aec30cbe904","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:08.898253959Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:08.89831362Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:08.898358081Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:08.898396281Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:08.898444382Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:08.901852023Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:08.906063916Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"region c9a9186a-9d34-472c-9bbc-198747552ab1 running snapshot 55708258-8abd-4c4b-8508-8aec30cbe904 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:08.906079016Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"datafile"} {"msg":"path is \"/data/regions/c9a9186a-9d34-472c-9bbc-198747552ab1\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:09.200186675Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"datafile"} {"msg":"dataset is oxp_16bb833e-d265-4e47-8c19-1cb9a8fa2175/crucible/regions/c9a9186a-9d34-472c-9bbc-198747552ab1","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:09.208937805Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"datafile"} {"msg":"snapshot is oxp_16bb833e-d265-4e47-8c19-1cb9a8fa2175/crucible/regions/c9a9186a-9d34-472c-9bbc-198747552ab1@55708258-8abd-4c4b-8508-8aec30cbe904","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:09.217138528Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_16bb833e-d265-4e47-8c19-1cb9a8fa2175/crucible/regions/c9a9186a-9d34-472c-9bbc-198747552ab1\", \"55708258-8abd-4c4b-8508-8aec30cbe904\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:09.217162318Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:09.225392971Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"uri":"/crucible/0/regions/c9a9186a-9d34-472c-9bbc-198747552ab1/snapshots/55708258-8abd-4c4b-8508-8aec30cbe904/run","method":"POST","req_id":"f4fbb4bf-a91b-4b96-b733-d2371f16a442","remote_addr":"[fd00:1122:3344:101::b]:48773","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":25238,"response_code":"200"} {"msg":"region cc4e200c-7f04-4d22-8319-b9ac1aa171ef state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.419077864Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.419273147Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"uri":"/crucible/0/regions","method":"POST","req_id":"c92aa69a-4977-4bb7-81a7-b22f09d404b3","remote_addr":"[fd00:1122:3344:101::b]:48773","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":869,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.419299597Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_16bb833e-d265-4e47-8c19-1cb9a8fa2175/crucible/regions/cc4e200c-7f04-4d22-8319-b9ac1aa171ef","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.429892195Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_16bb833e-d265-4e47-8c19-1cb9a8fa2175/crucible/regions/cc4e200c-7f04-4d22-8319-b9ac1aa171ef","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.429923586Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"cc4e200c-7f04-4d22-8319-b9ac1aa171ef\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/cc4e200c-7f04-4d22-8319-b9ac1aa171ef\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.494857275Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"region":"cc4e200c-7f04-4d22-8319-b9ac1aa171ef","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"cc4e200c-7f04-4d22-8319-b9ac1aa171ef\" \"--data\" \"/data/regions/cc4e200c-7f04-4d22-8319-b9ac1aa171ef\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.494922836Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"region":"cc4e200c-7f04-4d22-8319-b9ac1aa171ef","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.570223669Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"region":"cc4e200c-7f04-4d22-8319-b9ac1aa171ef","component":"worker"} {"msg":"region cc4e200c-7f04-4d22-8319-b9ac1aa171ef state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.57027265Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.570343371Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"existing directory value /dev/null does not match /data/regions/c9a9186a-9d34-472c-9bbc-198747552ab1","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.571917714Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.571999086Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.572080837Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.572088517Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.572126158Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-c9a9186a-9d34-472c-9bbc-198747552ab1","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.572170628Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c9a9186a-9d34-472c-9bbc-198747552ab1","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.572796068Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.572874579Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.57292685Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.57297514Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.579431787Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-cc4e200c-7f04-4d22-8319-b9ac1aa171ef","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.579897894Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-cc4e200c-7f04-4d22-8319-b9ac1aa171ef","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.584120797Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.591450186Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-cc4e200c-7f04-4d22-8319-b9ac1aa171ef","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.594030135Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/cc4e200c-7f04-4d22-8319-b9ac1aa171ef","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.594303179Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19003","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.594366649Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.59441708Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.594500351Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.598651804Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.603944303Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.756389427Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"uri":"/crucible/0/regions","method":"POST","req_id":"c2deb96c-db6e-433e-98b5-e0b444cd0c17","remote_addr":"[fd00:1122:3344:101::b]:48773","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":81,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.101509556Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"uri":"/crucible/0/regions/cc4e200c-7f04-4d22-8319-b9ac1aa171ef","method":"GET","req_id":"ecd1dd19-4317-4432-87e1-ea3860888531","remote_addr":"[fd00:1122:3344:101::b]:48773","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":52,"response_code":"200"} {"msg":"path is \"/data/regions/cc4e200c-7f04-4d22-8319-b9ac1aa171ef\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.105090529Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"datafile"} {"msg":"dataset is oxp_16bb833e-d265-4e47-8c19-1cb9a8fa2175/crucible/regions/cc4e200c-7f04-4d22-8319-b9ac1aa171ef","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.118019312Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"datafile"} {"msg":"region cc4e200c-7f04-4d22-8319-b9ac1aa171ef state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.127709257Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.127895329Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"disabling downstairs instance: downstairs-cc4e200c-7f04-4d22-8319-b9ac1aa171ef (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.129223989Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.130933065Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"uri":"/crucible/0/regions/cc4e200c-7f04-4d22-8319-b9ac1aa171ef","method":"DELETE","req_id":"f441d182-a9a1-402f-8613-652e69063821","remote_addr":"[fd00:1122:3344:101::b]:48773","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":25868,"response_code":"204"} {"msg":"existing directory value /dev/null does not match /data/regions/c9a9186a-9d34-472c-9bbc-198747552ab1","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.134572349Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.134706221Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.134824243Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.134834393Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.134879824Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-c9a9186a-9d34-472c-9bbc-198747552ab1","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.134934474Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.135375291Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"uri":"/crucible/0/regions/cc4e200c-7f04-4d22-8319-b9ac1aa171ef","method":"GET","req_id":"d802e415-c601-4444-b3b8-c2f5df93bdb1","remote_addr":"[fd00:1122:3344:101::b]:48773","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":35,"response_code":"200"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c9a9186a-9d34-472c-9bbc-198747552ab1","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.135445142Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.135506993Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.135557234Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.135605644Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.140771112Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.142106672Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_16bb833e-d265-4e47-8c19-1cb9a8fa2175/crucible/regions/cc4e200c-7f04-4d22-8319-b9ac1aa171ef\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.153183877Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"region":"cc4e200c-7f04-4d22-8319-b9ac1aa171ef","component":"worker"} {"msg":"region cc4e200c-7f04-4d22-8319-b9ac1aa171ef state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.20700234Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.262111982Z","hostname":"oxz_crucible_f0ab8838-1fca-48d0-8066-01e038a185fd","pid":22538,"uri":"/crucible/0/regions/cc4e200c-7f04-4d22-8319-b9ac1aa171ef","method":"GET","req_id":"6d661569-2d4d-4c47-a4f0-4b0fab102aea","remote_addr":"[fd00:1122:3344:101::b]:48773","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":71,"response_code":"200"}