[ Oct 14 10:09:56 Enabled. ]
[ Oct 14 10:09:56 Rereading configuration. ]
[ Oct 14 10:09:59 Rereading configuration. ]
[ Oct 14 10:10:05 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_9ad630be-5982-4ebb-8428-bca18a77200a/crucible -l [fd00:1122:3344:101::15]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_9ad630be-5982-4ebb-8428-bca18a77200a/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:10:05.87021357Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615}
{"msg":"listen IP: [fd00:1122:3344:101::15]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:10:05.87162828Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:10:05.87163615Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:10:05.887284175Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:10:05.979921031Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:02.071096507Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:45772"}
{"msg":"region d5896c47-d07e-4476-b61e-30ae98804303 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:02.118256137Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:02.127631072Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:02.127656023Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"uri":"/crucible/0/regions","method":"POST","req_id":"e6e55e1c-29fe-41c0-8406-40a3f7d86eca","remote_addr":"[fd00:1122:3344:101::a]:45772","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":18753,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_9ad630be-5982-4ebb-8428-bca18a77200a/crucible/regions/d5896c47-d07e-4476-b61e-30ae98804303","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:02.137829754Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_9ad630be-5982-4ebb-8428-bca18a77200a/crucible/regions/d5896c47-d07e-4476-b61e-30ae98804303","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:02.137866595Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"d5896c47-d07e-4476-b61e-30ae98804303\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/d5896c47-d07e-4476-b61e-30ae98804303\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:02.197446103Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"region":"d5896c47-d07e-4476-b61e-30ae98804303","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"d5896c47-d07e-4476-b61e-30ae98804303\" \"--data\" \"/data/regions/d5896c47-d07e-4476-b61e-30ae98804303\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:02.201663481Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"region":"d5896c47-d07e-4476-b61e-30ae98804303","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:02.363195548Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"uri":"/crucible/0/regions","method":"POST","req_id":"08a90dbc-8210-4446-8e17-deae13f65c45","remote_addr":"[fd00:1122:3344:101::a]:45772","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":80,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:02.547165542Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"region":"d5896c47-d07e-4476-b61e-30ae98804303","component":"worker"}
{"msg":"region d5896c47-d07e-4476-b61e-30ae98804303 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:02.547213103Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:02.547300755Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-d5896c47-d07e-4476-b61e-30ae98804303","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:02.549753446Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-d5896c47-d07e-4476-b61e-30ae98804303","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:02.557755662Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:02.610030869Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-d5896c47-d07e-4476-b61e-30ae98804303","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:02.62598311Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/d5896c47-d07e-4476-b61e-30ae98804303","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:02.626637424Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:02.626699445Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:02.626741466Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:02.626784267Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:02.633322583Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:02.63798087Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:03.055849505Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"uri":"/crucible/0/regions","method":"POST","req_id":"45a013da-263b-46d8-94f3-0f724b036f9b","remote_addr":"[fd00:1122:3344:101::a]:45772","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":54,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:03.532442131Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"uri":"/crucible/0/regions/d5896c47-d07e-4476-b61e-30ae98804303","method":"GET","req_id":"1fc0d8fc-ab84-4275-931d-764ade5885b0","remote_addr":"[fd00:1122:3344:101::a]:45772","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":914,"response_code":"200"}
{"msg":"path is \"/data/regions/d5896c47-d07e-4476-b61e-30ae98804303\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:03.534991704Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"datafile"}
{"msg":"dataset is oxp_9ad630be-5982-4ebb-8428-bca18a77200a/crucible/regions/d5896c47-d07e-4476-b61e-30ae98804303","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:03.54584421Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"datafile"}
{"msg":"region d5896c47-d07e-4476-b61e-30ae98804303 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:03.55596234Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:03.556116423Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:03.556190325Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"uri":"/crucible/0/regions/d5896c47-d07e-4476-b61e-30ae98804303","method":"DELETE","req_id":"34e072f3-da17-407d-855d-af14e24908ea","remote_addr":"[fd00:1122:3344:101::a]:45772","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":21229,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-d5896c47-d07e-4476-b61e-30ae98804303 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:03.557304438Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:03.558864591Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"uri":"/crucible/0/regions/d5896c47-d07e-4476-b61e-30ae98804303","method":"GET","req_id":"e81e2add-858c-477a-ac9d-074adfb04552","remote_addr":"[fd00:1122:3344:101::a]:45772","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":32,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:03.568122923Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_9ad630be-5982-4ebb-8428-bca18a77200a/crucible/regions/d5896c47-d07e-4476-b61e-30ae98804303\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:03.579184853Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"region":"d5896c47-d07e-4476-b61e-30ae98804303","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:03.676822623Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"uri":"/crucible/0/regions/d5896c47-d07e-4476-b61e-30ae98804303","method":"GET","req_id":"8017f7ed-ac4f-4625-9fd6-858318b2fa31","remote_addr":"[fd00:1122:3344:101::a]:45772","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":746,"response_code":"200"}
{"msg":"region d5896c47-d07e-4476-b61e-30ae98804303 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:03.680684453Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:03.816129248Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"uri":"/crucible/0/regions/d5896c47-d07e-4476-b61e-30ae98804303","method":"GET","req_id":"b5aa5f00-28e9-47f7-ad08-8eff619b8a19","remote_addr":"[fd00:1122:3344:101::a]:45772","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":29,"response_code":"200"}
{"msg":"region 5d4b3ed9-d254-4e46-a96d-ce1bf523f6fc state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:05.856803034Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:05.856966177Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"uri":"/crucible/0/regions","method":"POST","req_id":"0da11531-b233-4974-a24c-1780111dfdb8","remote_addr":"[fd00:1122:3344:101::a]:45772","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":229,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:05.856993327Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_9ad630be-5982-4ebb-8428-bca18a77200a/crucible/regions/5d4b3ed9-d254-4e46-a96d-ce1bf523f6fc","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:05.865197328Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_9ad630be-5982-4ebb-8428-bca18a77200a/crucible/regions/5d4b3ed9-d254-4e46-a96d-ce1bf523f6fc","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:05.865221449Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"5d4b3ed9-d254-4e46-a96d-ce1bf523f6fc\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/5d4b3ed9-d254-4e46-a96d-ce1bf523f6fc\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:05.937111403Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"region":"5d4b3ed9-d254-4e46-a96d-ce1bf523f6fc","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"5d4b3ed9-d254-4e46-a96d-ce1bf523f6fc\" \"--data\" \"/data/regions/5d4b3ed9-d254-4e46-a96d-ce1bf523f6fc\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:05.937167334Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"region":"5d4b3ed9-d254-4e46-a96d-ce1bf523f6fc","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:06.057845722Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"region":"5d4b3ed9-d254-4e46-a96d-ce1bf523f6fc","component":"worker"}
{"msg":"region 5d4b3ed9-d254-4e46-a96d-ce1bf523f6fc state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:06.057904423Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:06.057990405Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-5d4b3ed9-d254-4e46-a96d-ce1bf523f6fc","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:06.059570188Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-5d4b3ed9-d254-4e46-a96d-ce1bf523f6fc","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:06.062835756Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:06.07937819Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:06.08852521Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"uri":"/crucible/0/regions","method":"POST","req_id":"ab08a971-1a5e-4f1f-93dd-15ee64319af1","remote_addr":"[fd00:1122:3344:101::a]:45772","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":71,"response_code":"200"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-5d4b3ed9-d254-4e46-a96d-ce1bf523f6fc","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:06.089941069Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/5d4b3ed9-d254-4e46-a96d-ce1bf523f6fc","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:06.090144764Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:06.090193615Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:06.090233326Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:06.090272586Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:06.097875244Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:11:06.107067365Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:16.618075394Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:43882"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:16.618792689Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"uri":"/crucible/0/regions/5d4b3ed9-d254-4e46-a96d-ce1bf523f6fc","method":"GET","req_id":"353646da-bb44-4cf5-bc47-a9e6f74fee90","remote_addr":"[fd00:1122:3344:101::a]:43882","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":670,"response_code":"200"}
{"msg":"path is \"/data/regions/5d4b3ed9-d254-4e46-a96d-ce1bf523f6fc\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:16.621794511Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"datafile"}
{"msg":"dataset is oxp_9ad630be-5982-4ebb-8428-bca18a77200a/crucible/regions/5d4b3ed9-d254-4e46-a96d-ce1bf523f6fc","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:16.630718357Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"datafile"}
{"msg":"snapshot is oxp_9ad630be-5982-4ebb-8428-bca18a77200a/crucible/regions/5d4b3ed9-d254-4e46-a96d-ce1bf523f6fc@627e8e81-9ab8-4485-acfb-a4232d902510","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:16.638125141Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_9ad630be-5982-4ebb-8428-bca18a77200a/crucible/regions/5d4b3ed9-d254-4e46-a96d-ce1bf523f6fc\", \"627e8e81-9ab8-4485-acfb-a4232d902510\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:16.638160962Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:16.650256653Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"uri":"/crucible/0/regions/5d4b3ed9-d254-4e46-a96d-ce1bf523f6fc/snapshots/627e8e81-9ab8-4485-acfb-a4232d902510","method":"GET","req_id":"a52c13fb-ef96-495b-977d-c2b89981ad71","remote_addr":"[fd00:1122:3344:101::a]:43882","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":28471,"response_code":"200"}
{"msg":"path is \"/data/regions/5d4b3ed9-d254-4e46-a96d-ce1bf523f6fc\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:16.658366311Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"datafile"}
{"msg":"dataset is oxp_9ad630be-5982-4ebb-8428-bca18a77200a/crucible/regions/5d4b3ed9-d254-4e46-a96d-ce1bf523f6fc","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:16.666223805Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"datafile"}
{"msg":"snapshot is oxp_9ad630be-5982-4ebb-8428-bca18a77200a/crucible/regions/5d4b3ed9-d254-4e46-a96d-ce1bf523f6fc@627e8e81-9ab8-4485-acfb-a4232d902510","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:16.674344554Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_9ad630be-5982-4ebb-8428-bca18a77200a/crucible/regions/5d4b3ed9-d254-4e46-a96d-ce1bf523f6fc\", \"627e8e81-9ab8-4485-acfb-a4232d902510\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:16.674379175Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 5d4b3ed9-d254-4e46-a96d-ce1bf523f6fc-627e8e81-9ab8-4485-acfb-a4232d902510 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:16.690310915Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"datafile"}
{"msg":"applying SMF actions for region 5d4b3ed9-d254-4e46-a96d-ce1bf523f6fc running snapshot 627e8e81-9ab8-4485-acfb-a4232d902510 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:16.691033321Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:16.691079522Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"uri":"/crucible/0/regions/5d4b3ed9-d254-4e46-a96d-ce1bf523f6fc/snapshots/627e8e81-9ab8-4485-acfb-a4232d902510/run","method":"POST","req_id":"7d9d3773-0dc1-4867-bbb9-79648718b914","remote_addr":"[fd00:1122:3344:101::a]:43882","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":32748,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-5d4b3ed9-d254-4e46-a96d-ce1bf523f6fc-627e8e81-9ab8-4485-acfb-a4232d902510","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:16.700923396Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-5d4b3ed9-d254-4e46-a96d-ce1bf523f6fc-627e8e81-9ab8-4485-acfb-a4232d902510","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:16.709509765Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-5d4b3ed9-d254-4e46-a96d-ce1bf523f6fc-627e8e81-9ab8-4485-acfb-a4232d902510","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:16.726044918Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/5d4b3ed9-d254-4e46-a96d-ce1bf523f6fc/.zfs/snapshot/627e8e81-9ab8-4485-acfb-a4232d902510","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:16.726319934Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:16.726382935Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:16.726425366Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:16.726465547Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:16.726520228Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:16.733029763Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:16.740507139Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"region 5d4b3ed9-d254-4e46-a96d-ce1bf523f6fc running snapshot 627e8e81-9ab8-4485-acfb-a4232d902510 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:16.740523629Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"datafile"}
{"msg":"path is \"/data/regions/5d4b3ed9-d254-4e46-a96d-ce1bf523f6fc\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:16.842023629Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"datafile"}
{"msg":"dataset is oxp_9ad630be-5982-4ebb-8428-bca18a77200a/crucible/regions/5d4b3ed9-d254-4e46-a96d-ce1bf523f6fc","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:16.850491435Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"datafile"}
{"msg":"snapshot is oxp_9ad630be-5982-4ebb-8428-bca18a77200a/crucible/regions/5d4b3ed9-d254-4e46-a96d-ce1bf523f6fc@627e8e81-9ab8-4485-acfb-a4232d902510","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:16.858387249Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_9ad630be-5982-4ebb-8428-bca18a77200a/crucible/regions/5d4b3ed9-d254-4e46-a96d-ce1bf523f6fc\", \"627e8e81-9ab8-4485-acfb-a4232d902510\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:16.85842642Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:16.865706701Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"uri":"/crucible/0/regions/5d4b3ed9-d254-4e46-a96d-ce1bf523f6fc/snapshots/627e8e81-9ab8-4485-acfb-a4232d902510/run","method":"POST","req_id":"e788bcb2-743b-4cfe-895a-e5bfa92e80b7","remote_addr":"[fd00:1122:3344:101::a]:43882","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":23717,"response_code":"200"}
{"msg":"region e54f7cdd-65e5-49b3-a4f4-8073c30bca2f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.36692683Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.367111193Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"uri":"/crucible/0/regions","method":"POST","req_id":"1071adb6-de7d-417d-8d3f-a78aeb72a181","remote_addr":"[fd00:1122:3344:101::a]:43882","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":261,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.367156094Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_9ad630be-5982-4ebb-8428-bca18a77200a/crucible/regions/e54f7cdd-65e5-49b3-a4f4-8073c30bca2f","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.376143301Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_9ad630be-5982-4ebb-8428-bca18a77200a/crucible/regions/e54f7cdd-65e5-49b3-a4f4-8073c30bca2f","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.376180242Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"e54f7cdd-65e5-49b3-a4f4-8073c30bca2f\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/e54f7cdd-65e5-49b3-a4f4-8073c30bca2f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.443424Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"region":"e54f7cdd-65e5-49b3-a4f4-8073c30bca2f","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"e54f7cdd-65e5-49b3-a4f4-8073c30bca2f\" \"--data\" \"/data/regions/e54f7cdd-65e5-49b3-a4f4-8073c30bca2f\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.443484151Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"region":"e54f7cdd-65e5-49b3-a4f4-8073c30bca2f","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.507790647Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"uri":"/crucible/0/regions","method":"POST","req_id":"dc993346-f872-4f56-949b-a3fb04e3379c","remote_addr":"[fd00:1122:3344:101::a]:43882","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":91,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.558007781Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"region":"e54f7cdd-65e5-49b3-a4f4-8073c30bca2f","component":"worker"}
{"msg":"region e54f7cdd-65e5-49b3-a4f4-8073c30bca2f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.558030032Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.558146504Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-e54f7cdd-65e5-49b3-a4f4-8073c30bca2f","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.559973392Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-e54f7cdd-65e5-49b3-a4f4-8073c30bca2f","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.564750561Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.578198291Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e54f7cdd-65e5-49b3-a4f4-8073c30bca2f","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.581599452Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e54f7cdd-65e5-49b3-a4f4-8073c30bca2f","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.581805176Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.581853967Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.581893378Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.581930908Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.587407932Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.598452392Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.834213342Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"uri":"/crucible/0/regions","method":"POST","req_id":"c0284132-db7f-4032-9161-c878226cfe7f","remote_addr":"[fd00:1122:3344:101::a]:43882","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":58,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:47.5511835Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"uri":"/crucible/0/regions/e54f7cdd-65e5-49b3-a4f4-8073c30bca2f","method":"GET","req_id":"b947d0b5-6ef0-488d-896c-79136ddb82ef","remote_addr":"[fd00:1122:3344:101::a]:43882","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":812,"response_code":"200"}
{"msg":"path is \"/data/regions/e54f7cdd-65e5-49b3-a4f4-8073c30bca2f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:47.55407208Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"datafile"}
{"msg":"dataset is oxp_9ad630be-5982-4ebb-8428-bca18a77200a/crucible/regions/e54f7cdd-65e5-49b3-a4f4-8073c30bca2f","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:47.567894067Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"datafile"}
{"msg":"region e54f7cdd-65e5-49b3-a4f4-8073c30bca2f state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:47.57764982Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:47.577858274Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:47.577926346Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"uri":"/crucible/0/regions/e54f7cdd-65e5-49b3-a4f4-8073c30bca2f","method":"DELETE","req_id":"aa9fc2f5-c9d1-401e-9f12-e0474b40c9b6","remote_addr":"[fd00:1122:3344:101::a]:43882","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":23875,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-e54f7cdd-65e5-49b3-a4f4-8073c30bca2f (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:47.579624831Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:47.58198378Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"uri":"/crucible/0/regions/e54f7cdd-65e5-49b3-a4f4-8073c30bca2f","method":"GET","req_id":"54107105-ae62-4ff9-8631-0771f0856375","remote_addr":"[fd00:1122:3344:101::a]:43882","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":221,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:47.585557804Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_9ad630be-5982-4ebb-8428-bca18a77200a/crucible/regions/e54f7cdd-65e5-49b3-a4f4-8073c30bca2f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:47.597545763Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"region":"e54f7cdd-65e5-49b3-a4f4-8073c30bca2f","component":"worker"}
{"msg":"region e54f7cdd-65e5-49b3-a4f4-8073c30bca2f state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:47.655845645Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:47.694611441Z","hostname":"oxz_crucible_27efaf03-a6a5-45b5-ae0c-fd4020b002c4","pid":22615,"uri":"/crucible/0/regions/e54f7cdd-65e5-49b3-a4f4-8073c30bca2f","method":"GET","req_id":"97372eba-5344-4ff3-ac65-6efc918639a5","remote_addr":"[fd00:1122:3344:101::a]:43882","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":79,"response_code":"200"}