[ May  8 06:16:49 Enabled. ]
[ May  8 06:16:49 Rereading configuration. ]
[ May  8 06:16:52 Rereading configuration. ]
[ May  8 06:16:57 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_d19fb713-35d3-443d-bcf1-f5d6805f03bb/crucible -l [fd00:1122:3344:101::17]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_d19fb713-35d3-443d-bcf1-f5d6805f03bb/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:16:57.301727303Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027}
{"msg":"listen IP: [fd00:1122:3344:101::17]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:16:57.308209175Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:16:57.308219275Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:16:57.326294379Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:16:57.415778139Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:46.939342435Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:51271"}
{"msg":"region 9c425136-b67d-4f96-ac00-adf688e908a8 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:46.970775128Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:46.971395477Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"uri":"/crucible/0/regions","method":"POST","req_id":"a3c10cb4-2761-4f52-b46a-797ba15ffdc4","remote_addr":"[fd00:1122:3344:101::b]:51271","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":13669,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:46.973957102Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_d19fb713-35d3-443d-bcf1-f5d6805f03bb/crucible/regions/9c425136-b67d-4f96-ac00-adf688e908a8","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:46.983777361Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_d19fb713-35d3-443d-bcf1-f5d6805f03bb/crucible/regions/9c425136-b67d-4f96-ac00-adf688e908a8","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:46.983808121Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"9c425136-b67d-4f96-ac00-adf688e908a8\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/9c425136-b67d-4f96-ac00-adf688e908a8\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:47.033780644Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"region":"9c425136-b67d-4f96-ac00-adf688e908a8","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"9c425136-b67d-4f96-ac00-adf688e908a8\" \"--data\" \"/data/regions/9c425136-b67d-4f96-ac00-adf688e908a8\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:47.033840725Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"region":"9c425136-b67d-4f96-ac00-adf688e908a8","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:47.157250702Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"uri":"/crucible/0/regions","method":"POST","req_id":"f031bee3-d0f9-462e-bd5f-7deeea3beab5","remote_addr":"[fd00:1122:3344:101::b]:51271","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":70,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:47.370607834Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"region":"9c425136-b67d-4f96-ac00-adf688e908a8","component":"worker"}
{"msg":"region 9c425136-b67d-4f96-ac00-adf688e908a8 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:47.370661945Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:47.370752796Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-9c425136-b67d-4f96-ac00-adf688e908a8","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:47.371147932Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-9c425136-b67d-4f96-ac00-adf688e908a8","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:47.379041803Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:47.397246339Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-9c425136-b67d-4f96-ac00-adf688e908a8","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:47.400076039Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/9c425136-b67d-4f96-ac00-adf688e908a8","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:47.400821489Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:47.400966502Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:47.401013732Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:47.401054983Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:47.404654193Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:47.414031735Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:17:47.434570074Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"uri":"/crucible/0/regions","method":"POST","req_id":"66dc912e-9f98-46bd-8cef-96eb058305fa","remote_addr":"[fd00:1122:3344:101::b]:51271","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":53,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:42.943195291Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:33654"}
{"msg":"region ed69ddbd-aed6-4eeb-8d9e-4a2239704199 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:42.943918531Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:42.944004782Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:42.944019873Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"uri":"/crucible/0/regions","method":"POST","req_id":"fa2622bf-c886-4041-94a4-7d98c61fc6b0","remote_addr":"[fd00:1122:3344:101::b]:33654","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":749,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_d19fb713-35d3-443d-bcf1-f5d6805f03bb/crucible/regions/ed69ddbd-aed6-4eeb-8d9e-4a2239704199","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:42.954054004Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_d19fb713-35d3-443d-bcf1-f5d6805f03bb/crucible/regions/ed69ddbd-aed6-4eeb-8d9e-4a2239704199","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:42.954096824Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"ed69ddbd-aed6-4eeb-8d9e-4a2239704199\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/ed69ddbd-aed6-4eeb-8d9e-4a2239704199\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:43.017370655Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"region":"ed69ddbd-aed6-4eeb-8d9e-4a2239704199","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"ed69ddbd-aed6-4eeb-8d9e-4a2239704199\" \"--data\" \"/data/regions/ed69ddbd-aed6-4eeb-8d9e-4a2239704199\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:43.01988648Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"region":"ed69ddbd-aed6-4eeb-8d9e-4a2239704199","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:43.094723683Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"region":"ed69ddbd-aed6-4eeb-8d9e-4a2239704199","component":"worker"}
{"msg":"region ed69ddbd-aed6-4eeb-8d9e-4a2239704199 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:43.094754354Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:43.094886456Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-ed69ddbd-aed6-4eeb-8d9e-4a2239704199","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:43.110633617Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-ed69ddbd-aed6-4eeb-8d9e-4a2239704199","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:43.116806554Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:43.126659103Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-ed69ddbd-aed6-4eeb-8d9e-4a2239704199","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:43.129572254Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ed69ddbd-aed6-4eeb-8d9e-4a2239704199","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:43.129766307Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:43.129817127Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:43.129855218Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:43.129891378Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:43.13357225Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:43.137590077Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:43.299380633Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"uri":"/crucible/0/regions","method":"POST","req_id":"9b5db207-ff66-493c-9307-27b63cb5fd63","remote_addr":"[fd00:1122:3344:101::b]:33654","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":60,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.544334822Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"uri":"/crucible/0/regions/9c425136-b67d-4f96-ac00-adf688e908a8","method":"GET","req_id":"671edbd9-2832-43a8-ae52-fa3e31255aa5","remote_addr":"[fd00:1122:3344:101::b]:33654","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":54,"response_code":"200"}
{"msg":"path is \"/data/regions/9c425136-b67d-4f96-ac00-adf688e908a8\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.547411206Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"datafile"}
{"msg":"dataset is oxp_d19fb713-35d3-443d-bcf1-f5d6805f03bb/crucible/regions/9c425136-b67d-4f96-ac00-adf688e908a8","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.556909659Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"datafile"}
{"msg":"snapshot is oxp_d19fb713-35d3-443d-bcf1-f5d6805f03bb/crucible/regions/9c425136-b67d-4f96-ac00-adf688e908a8@32c9588a-9338-4f7d-8835-b51c992dbfc6","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.565372689Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_d19fb713-35d3-443d-bcf1-f5d6805f03bb/crucible/regions/9c425136-b67d-4f96-ac00-adf688e908a8\", \"32c9588a-9338-4f7d-8835-b51c992dbfc6\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.565400879Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.577399018Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"uri":"/crucible/0/regions/9c425136-b67d-4f96-ac00-adf688e908a8/snapshots/32c9588a-9338-4f7d-8835-b51c992dbfc6","method":"GET","req_id":"b8a8e8b0-1fc1-4611-8009-1b8290314aa4","remote_addr":"[fd00:1122:3344:101::b]:33654","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":29986,"response_code":"200"}
{"msg":"path is \"/data/regions/9c425136-b67d-4f96-ac00-adf688e908a8\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.580826266Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"datafile"}
{"msg":"dataset is oxp_d19fb713-35d3-443d-bcf1-f5d6805f03bb/crucible/regions/9c425136-b67d-4f96-ac00-adf688e908a8","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.589254214Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"datafile"}
{"msg":"snapshot is oxp_d19fb713-35d3-443d-bcf1-f5d6805f03bb/crucible/regions/9c425136-b67d-4f96-ac00-adf688e908a8@32c9588a-9338-4f7d-8835-b51c992dbfc6","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.596804021Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_d19fb713-35d3-443d-bcf1-f5d6805f03bb/crucible/regions/9c425136-b67d-4f96-ac00-adf688e908a8\", \"32c9588a-9338-4f7d-8835-b51c992dbfc6\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.596838811Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 9c425136-b67d-4f96-ac00-adf688e908a8-32c9588a-9338-4f7d-8835-b51c992dbfc6 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.607063635Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"datafile"}
{"msg":"applying SMF actions for region 9c425136-b67d-4f96-ac00-adf688e908a8 running snapshot 32c9588a-9338-4f7d-8835-b51c992dbfc6 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.607279628Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.607293478Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"uri":"/crucible/0/regions/9c425136-b67d-4f96-ac00-adf688e908a8/snapshots/32c9588a-9338-4f7d-8835-b51c992dbfc6/run","method":"POST","req_id":"d3b489c4-3420-48b4-9f7b-ee56939e16ca","remote_addr":"[fd00:1122:3344:101::b]:33654","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":26483,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-9c425136-b67d-4f96-ac00-adf688e908a8-32c9588a-9338-4f7d-8835-b51c992dbfc6","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.6095281Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-9c425136-b67d-4f96-ac00-adf688e908a8-32c9588a-9338-4f7d-8835-b51c992dbfc6","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.619775344Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-9c425136-b67d-4f96-ac00-adf688e908a8-32c9588a-9338-4f7d-8835-b51c992dbfc6","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.629698264Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/9c425136-b67d-4f96-ac00-adf688e908a8/.zfs/snapshot/32c9588a-9338-4f7d-8835-b51c992dbfc6","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.630014048Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.630076849Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.63012139Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.63016247Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.630221201Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.634153416Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.639068935Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"region 9c425136-b67d-4f96-ac00-adf688e908a8 running snapshot 32c9588a-9338-4f7d-8835-b51c992dbfc6 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.639084446Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"datafile"}
{"msg":"path is \"/data/regions/9c425136-b67d-4f96-ac00-adf688e908a8\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.753431595Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"datafile"}
{"msg":"dataset is oxp_d19fb713-35d3-443d-bcf1-f5d6805f03bb/crucible/regions/9c425136-b67d-4f96-ac00-adf688e908a8","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.762478852Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"datafile"}
{"msg":"snapshot is oxp_d19fb713-35d3-443d-bcf1-f5d6805f03bb/crucible/regions/9c425136-b67d-4f96-ac00-adf688e908a8@32c9588a-9338-4f7d-8835-b51c992dbfc6","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.770764859Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_d19fb713-35d3-443d-bcf1-f5d6805f03bb/crucible/regions/9c425136-b67d-4f96-ac00-adf688e908a8\", \"32c9588a-9338-4f7d-8835-b51c992dbfc6\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.770805859Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:44.779017115Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"uri":"/crucible/0/regions/9c425136-b67d-4f96-ac00-adf688e908a8/snapshots/32c9588a-9338-4f7d-8835-b51c992dbfc6/run","method":"POST","req_id":"3676f948-32e7-43c3-b7fd-5206b0d2a7a5","remote_addr":"[fd00:1122:3344:101::b]:33654","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":25582,"response_code":"200"}
{"msg":"region e54717d1-7734-427c-9e50-8765b1b03cd9 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:48.909614609Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:48.909810212Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"uri":"/crucible/0/regions","method":"POST","req_id":"d80035a3-e440-4978-9d36-9e4376f267f3","remote_addr":"[fd00:1122:3344:101::b]:33654","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":274,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:48.909854543Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_d19fb713-35d3-443d-bcf1-f5d6805f03bb/crucible/regions/e54717d1-7734-427c-9e50-8765b1b03cd9","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:48.920110357Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_d19fb713-35d3-443d-bcf1-f5d6805f03bb/crucible/regions/e54717d1-7734-427c-9e50-8765b1b03cd9","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:48.920139917Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"e54717d1-7734-427c-9e50-8765b1b03cd9\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/e54717d1-7734-427c-9e50-8765b1b03cd9\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:48.969277969Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"region":"e54717d1-7734-427c-9e50-8765b1b03cd9","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"e54717d1-7734-427c-9e50-8765b1b03cd9\" \"--data\" \"/data/regions/e54717d1-7734-427c-9e50-8765b1b03cd9\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:48.971218506Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"region":"e54717d1-7734-427c-9e50-8765b1b03cd9","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:49.047043313Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"region":"e54717d1-7734-427c-9e50-8765b1b03cd9","component":"worker"}
{"msg":"region e54717d1-7734-427c-9e50-8765b1b03cd9 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:49.047080734Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:49.047210825Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-e54717d1-7734-427c-9e50-8765b1b03cd9","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:49.048495004Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-e54717d1-7734-427c-9e50-8765b1b03cd9","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:49.051445145Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:49.057986357Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e54717d1-7734-427c-9e50-8765b1b03cd9","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:49.061160802Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e54717d1-7734-427c-9e50-8765b1b03cd9","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:49.061433306Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19003","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:49.061513247Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:49.061590778Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:49.061650369Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:49.064958115Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:49.076989185Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:28:49.11006211Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"uri":"/crucible/0/regions","method":"POST","req_id":"405b09ba-34a7-4125-ad65-c117fcfc8fb1","remote_addr":"[fd00:1122:3344:101::b]:33654","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":76,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:29:14.437806827Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"uri":"/crucible/0/regions/e54717d1-7734-427c-9e50-8765b1b03cd9","method":"GET","req_id":"ada739fa-651d-48ad-bf47-f2abb975bcaa","remote_addr":"[fd00:1122:3344:101::b]:33654","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":48,"response_code":"200"}
{"msg":"path is \"/data/regions/e54717d1-7734-427c-9e50-8765b1b03cd9\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:29:14.441670272Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"datafile"}
{"msg":"dataset is oxp_d19fb713-35d3-443d-bcf1-f5d6805f03bb/crucible/regions/e54717d1-7734-427c-9e50-8765b1b03cd9","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:29:14.455560867Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"datafile"}
{"msg":"region e54717d1-7734-427c-9e50-8765b1b03cd9 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:29:14.465162582Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:29:14.465915803Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:29:14.465941713Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"uri":"/crucible/0/regions/e54717d1-7734-427c-9e50-8765b1b03cd9","method":"DELETE","req_id":"258202e7-37d2-42ba-b3ed-d5d8b66c418f","remote_addr":"[fd00:1122:3344:101::b]:33654","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":24301,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-e54717d1-7734-427c-9e50-8765b1b03cd9 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:29:14.46713451Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:29:14.470465277Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"uri":"/crucible/0/regions/e54717d1-7734-427c-9e50-8765b1b03cd9","method":"GET","req_id":"08ee2af7-cb0a-401d-aeda-17ea9521ec7c","remote_addr":"[fd00:1122:3344:101::b]:33654","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":25,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:29:14.475049941Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_d19fb713-35d3-443d-bcf1-f5d6805f03bb/crucible/regions/e54717d1-7734-427c-9e50-8765b1b03cd9\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:29:14.486019586Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"region":"e54717d1-7734-427c-9e50-8765b1b03cd9","component":"worker"}
{"msg":"region e54717d1-7734-427c-9e50-8765b1b03cd9 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:29:14.535289869Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T06:29:14.55380526Z","hostname":"oxz_crucible_712f33cf-6f39-443c-b846-779336b8c9c7","pid":26027,"uri":"/crucible/0/regions/e54717d1-7734-427c-9e50-8765b1b03cd9","method":"GET","req_id":"e5ad15d7-9994-46b4-82ae-295f9066cfb1","remote_addr":"[fd00:1122:3344:101::b]:33654","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":64,"response_code":"200"}