[ Apr 28 18:09:59 Enabled. ]
[ Apr 28 18:09:59 Rereading configuration. ]
[ Apr 28 18:10:04 Rereading configuration. ]
[ Apr 28 18:10:10 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_0fba72b0-54a2-409e-8c16-b7895f9546b4/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_0fba72b0-54a2-409e-8c16-b7895f9546b4/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:10.969481102Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971}
{"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:10.972883754Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:10.972892304Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:10.990487262Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:11.112523759Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:41.636359102Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:45598"}
{"msg":"region 69325bcf-f488-45f9-8c83-c01127ba55c2 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:41.67029781Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:41.675346145Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"uri":"/crucible/0/regions","method":"POST","req_id":"4b1e6494-79a1-49a6-9d07-acfdc86b9a15","remote_addr":"[fd00:1122:3344:101::c]:45598","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":18612,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:41.67765076Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_0fba72b0-54a2-409e-8c16-b7895f9546b4/crucible/regions/69325bcf-f488-45f9-8c83-c01127ba55c2","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:41.687353758Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_0fba72b0-54a2-409e-8c16-b7895f9546b4/crucible/regions/69325bcf-f488-45f9-8c83-c01127ba55c2","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:41.687414869Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"69325bcf-f488-45f9-8c83-c01127ba55c2\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/69325bcf-f488-45f9-8c83-c01127ba55c2\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:41.760835966Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"region":"69325bcf-f488-45f9-8c83-c01127ba55c2","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"69325bcf-f488-45f9-8c83-c01127ba55c2\" \"--data\" \"/data/regions/69325bcf-f488-45f9-8c83-c01127ba55c2\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:41.760922547Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"region":"69325bcf-f488-45f9-8c83-c01127ba55c2","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:41.97930444Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"uri":"/crucible/0/regions","method":"POST","req_id":"4dc282ae-4bca-435b-80da-154dc1dd4f5f","remote_addr":"[fd00:1122:3344:101::c]:45598","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":68,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:42.057083384Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"region":"69325bcf-f488-45f9-8c83-c01127ba55c2","component":"worker"}
{"msg":"region 69325bcf-f488-45f9-8c83-c01127ba55c2 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:42.057215296Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:42.05745677Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-69325bcf-f488-45f9-8c83-c01127ba55c2","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:42.057862586Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-69325bcf-f488-45f9-8c83-c01127ba55c2","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:42.064210083Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:42.081396964Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-69325bcf-f488-45f9-8c83-c01127ba55c2","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:42.093566529Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/69325bcf-f488-45f9-8c83-c01127ba55c2","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:42.094160468Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:42.09425067Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:42.094308861Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:42.094350551Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:42.099813364Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:42.106653058Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:42.715546034Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"uri":"/crucible/0/regions","method":"POST","req_id":"6f0e03b2-9eee-4148-a7f8-7f61515add78","remote_addr":"[fd00:1122:3344:101::c]:45598","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":44,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:42.971958025Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"uri":"/crucible/0/regions/69325bcf-f488-45f9-8c83-c01127ba55c2","method":"GET","req_id":"505471e0-6d61-4d38-b885-9b8e61e6d918","remote_addr":"[fd00:1122:3344:101::c]:45598","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":77,"response_code":"200"}
{"msg":"path is \"/data/regions/69325bcf-f488-45f9-8c83-c01127ba55c2\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:42.974762018Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"component":"datafile"}
{"msg":"dataset is oxp_0fba72b0-54a2-409e-8c16-b7895f9546b4/crucible/regions/69325bcf-f488-45f9-8c83-c01127ba55c2","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:42.986203742Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"component":"datafile"}
{"msg":"region 69325bcf-f488-45f9-8c83-c01127ba55c2 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:42.997046837Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:42.997274061Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:42.997328801Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"uri":"/crucible/0/regions/69325bcf-f488-45f9-8c83-c01127ba55c2","method":"DELETE","req_id":"fe6b2ba7-f622-470c-a08e-dbc806a12aea","remote_addr":"[fd00:1122:3344:101::c]:45598","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":22611,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-69325bcf-f488-45f9-8c83-c01127ba55c2 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:42.998347537Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:43.000990257Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"uri":"/crucible/0/regions/69325bcf-f488-45f9-8c83-c01127ba55c2","method":"GET","req_id":"b926a349-fe59-45ee-a01e-88bc12a31806","remote_addr":"[fd00:1122:3344:101::c]:45598","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":28,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:43.019082882Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_0fba72b0-54a2-409e-8c16-b7895f9546b4/crucible/regions/69325bcf-f488-45f9-8c83-c01127ba55c2\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:43.03141536Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"region":"69325bcf-f488-45f9-8c83-c01127ba55c2","component":"worker"}
{"msg":"region 69325bcf-f488-45f9-8c83-c01127ba55c2 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:43.112103078Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:43.120935374Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"uri":"/crucible/0/regions/69325bcf-f488-45f9-8c83-c01127ba55c2","method":"GET","req_id":"29244ef5-2b85-48ef-b49a-6fd3fa0b7a2a","remote_addr":"[fd00:1122:3344:101::c]:45598","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":60,"response_code":"200"}
{"msg":"region 3ee500b8-e212-4279-b25f-88b8c3e17cd6 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:44.706716763Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:44.706901045Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"uri":"/crucible/0/regions","method":"POST","req_id":"79d60db1-bd32-45c4-8bdc-a97f9dfc04e7","remote_addr":"[fd00:1122:3344:101::c]:45598","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":248,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:44.706952916Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_0fba72b0-54a2-409e-8c16-b7895f9546b4/crucible/regions/3ee500b8-e212-4279-b25f-88b8c3e17cd6","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:44.715655221Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_0fba72b0-54a2-409e-8c16-b7895f9546b4/crucible/regions/3ee500b8-e212-4279-b25f-88b8c3e17cd6","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:44.715690471Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"3ee500b8-e212-4279-b25f-88b8c3e17cd6\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/3ee500b8-e212-4279-b25f-88b8c3e17cd6\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:44.785780616Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"region":"3ee500b8-e212-4279-b25f-88b8c3e17cd6","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"3ee500b8-e212-4279-b25f-88b8c3e17cd6\" \"--data\" \"/data/regions/3ee500b8-e212-4279-b25f-88b8c3e17cd6\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:44.787916128Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"region":"3ee500b8-e212-4279-b25f-88b8c3e17cd6","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:44.866469224Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"region":"3ee500b8-e212-4279-b25f-88b8c3e17cd6","component":"worker"}
{"msg":"region 3ee500b8-e212-4279-b25f-88b8c3e17cd6 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:44.866536535Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:44.866756988Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-3ee500b8-e212-4279-b25f-88b8c3e17cd6","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:44.868302092Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-3ee500b8-e212-4279-b25f-88b8c3e17cd6","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:44.872101759Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:44.880598199Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-3ee500b8-e212-4279-b25f-88b8c3e17cd6","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:44.884553249Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3ee500b8-e212-4279-b25f-88b8c3e17cd6","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:44.884805913Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:44.884867813Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:44.884906314Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:44.884970395Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:44.88857964Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:44.893680798Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-28T18:10:44.966023879Z","hostname":"oxz_crucible_67629435-9e0f-42e9-837f-a57b1227716b","pid":22971,"uri":"/crucible/0/regions","method":"POST","req_id":"79440657-1225-4445-9ee4-e91e5a7b7c8f","remote_addr":"[fd00:1122:3344:101::c]:45598","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":54,"response_code":"200"}