[ Apr 22 03:02:31 Enabled. ]
[ Apr 22 03:02:31 Rereading configuration. ]
[ Apr 22 03:02:36 Rereading configuration. ]
[ Apr 22 03:02:41 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_8acca6af-d2ba-40aa-a651-5a9c30020d71/crucible -l [fd00:1122:3344:101::17]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_8acca6af-d2ba-40aa-a651-5a9c30020d71/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:02:41.773442994Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171}
{"msg":"listen IP: [fd00:1122:3344:101::17]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:02:41.776604478Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:02:41.776613358Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:02:41.795122957Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:02:41.92398629Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:03:15.146673244Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:61720"}
{"msg":"region 4570144a-fdc4-4f97-ac49-4899197679e3 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:03:15.188588475Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:03:15.193357433Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"uri":"/crucible/0/regions","method":"POST","req_id":"b8e673f6-2213-4c48-87c4-5229bea683be","remote_addr":"[fd00:1122:3344:101::b]:61720","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":18393,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:03:15.195758017Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_8acca6af-d2ba-40aa-a651-5a9c30020d71/crucible/regions/4570144a-fdc4-4f97-ac49-4899197679e3","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:03:15.205782795Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_8acca6af-d2ba-40aa-a651-5a9c30020d71/crucible/regions/4570144a-fdc4-4f97-ac49-4899197679e3","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:03:15.205820606Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"4570144a-fdc4-4f97-ac49-4899197679e3\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/4570144a-fdc4-4f97-ac49-4899197679e3\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:03:15.261793603Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"region":"4570144a-fdc4-4f97-ac49-4899197679e3","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"4570144a-fdc4-4f97-ac49-4899197679e3\" \"--data\" \"/data/regions/4570144a-fdc4-4f97-ac49-4899197679e3\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:03:15.264503686Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"region":"4570144a-fdc4-4f97-ac49-4899197679e3","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:03:15.556739642Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"uri":"/crucible/0/regions","method":"POST","req_id":"2f023111-5955-4442-9cdc-afb9ca4d1095","remote_addr":"[fd00:1122:3344:101::b]:61720","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:03:15.611323032Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"region":"4570144a-fdc4-4f97-ac49-4899197679e3","component":"worker"}
{"msg":"region 4570144a-fdc4-4f97-ac49-4899197679e3 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:03:15.611358583Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:03:15.611464126Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-4570144a-fdc4-4f97-ac49-4899197679e3","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:03:15.611875907Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-4570144a-fdc4-4f97-ac49-4899197679e3","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:03:15.618283268Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:03:15.635673553Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-4570144a-fdc4-4f97-ac49-4899197679e3","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:03:15.644280774Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/4570144a-fdc4-4f97-ac49-4899197679e3","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:03:15.653236723Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:03:15.653338646Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:03:15.653399888Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:03:15.653456259Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:03:15.657083866Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:03:15.663906799Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:03:16.054067184Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"uri":"/crucible/0/regions","method":"POST","req_id":"c7754508-9cea-4bc4-bca7-92ba183c8072","remote_addr":"[fd00:1122:3344:101::b]:61720","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":51,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:50.144868439Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:53342"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:50.148489956Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"uri":"/crucible/0/regions/4570144a-fdc4-4f97-ac49-4899197679e3","method":"GET","req_id":"bf63955c-38d3-45b4-87ea-f22fd339239a","remote_addr":"[fd00:1122:3344:101::b]:53342","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":3573,"response_code":"200"}
{"msg":"path is \"/data/regions/4570144a-fdc4-4f97-ac49-4899197679e3\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:50.151428774Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"datafile"}
{"msg":"dataset is oxp_8acca6af-d2ba-40aa-a651-5a9c30020d71/crucible/regions/4570144a-fdc4-4f97-ac49-4899197679e3","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:50.160908928Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"datafile"}
{"msg":"snapshot is oxp_8acca6af-d2ba-40aa-a651-5a9c30020d71/crucible/regions/4570144a-fdc4-4f97-ac49-4899197679e3@215b9cb3-2af4-4667-a31f-61646d6a4191","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:50.170092974Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_8acca6af-d2ba-40aa-a651-5a9c30020d71/crucible/regions/4570144a-fdc4-4f97-ac49-4899197679e3\", \"215b9cb3-2af4-4667-a31f-61646d6a4191\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:50.170123174Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:50.181129239Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"uri":"/crucible/0/regions/4570144a-fdc4-4f97-ac49-4899197679e3/snapshots/215b9cb3-2af4-4667-a31f-61646d6a4191","method":"GET","req_id":"02776294-0bae-4fe9-9d23-be9f6f86473e","remote_addr":"[fd00:1122:3344:101::b]:53342","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":29717,"response_code":"200"}
{"msg":"path is \"/data/regions/4570144a-fdc4-4f97-ac49-4899197679e3\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:50.184422907Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"datafile"}
{"msg":"dataset is oxp_8acca6af-d2ba-40aa-a651-5a9c30020d71/crucible/regions/4570144a-fdc4-4f97-ac49-4899197679e3","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:50.192558605Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"datafile"}
{"msg":"snapshot is oxp_8acca6af-d2ba-40aa-a651-5a9c30020d71/crucible/regions/4570144a-fdc4-4f97-ac49-4899197679e3@215b9cb3-2af4-4667-a31f-61646d6a4191","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:50.200206629Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_8acca6af-d2ba-40aa-a651-5a9c30020d71/crucible/regions/4570144a-fdc4-4f97-ac49-4899197679e3\", \"215b9cb3-2af4-4667-a31f-61646d6a4191\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:50.20024892Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 4570144a-fdc4-4f97-ac49-4899197679e3-215b9cb3-2af4-4667-a31f-61646d6a4191 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:50.209094047Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"datafile"}
{"msg":"applying SMF actions for region 4570144a-fdc4-4f97-ac49-4899197679e3 running snapshot 215b9cb3-2af4-4667-a31f-61646d6a4191 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:50.209713384Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:50.209733344Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"uri":"/crucible/0/regions/4570144a-fdc4-4f97-ac49-4899197679e3/snapshots/215b9cb3-2af4-4667-a31f-61646d6a4191/run","method":"POST","req_id":"39b121a2-ff89-4b87-b9a7-80a8d0831f18","remote_addr":"[fd00:1122:3344:101::b]:53342","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":25343,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/4570144a-fdc4-4f97-ac49-4899197679e3","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:50.225326241Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:50.225413353Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:50.225497556Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:50.225512726Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:50.225540147Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-4570144a-fdc4-4f97-ac49-4899197679e3","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:50.225596708Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/4570144a-fdc4-4f97-ac49-4899197679e3","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:50.225707191Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:50.225786213Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:50.225846135Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:50.225880826Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:50.233287174Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-4570144a-fdc4-4f97-ac49-4899197679e3-215b9cb3-2af4-4667-a31f-61646d6a4191","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:50.233744946Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-4570144a-fdc4-4f97-ac49-4899197679e3-215b9cb3-2af4-4667-a31f-61646d6a4191","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:50.236561802Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-4570144a-fdc4-4f97-ac49-4899197679e3-215b9cb3-2af4-4667-a31f-61646d6a4191","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:50.246026265Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/4570144a-fdc4-4f97-ac49-4899197679e3/.zfs/snapshot/215b9cb3-2af4-4667-a31f-61646d6a4191","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:50.246283112Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:50.246351444Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:50.246394035Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:50.246438376Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:50.246498808Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:50.250540456Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:50.255060517Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"region 4570144a-fdc4-4f97-ac49-4899197679e3 running snapshot 215b9cb3-2af4-4667-a31f-61646d6a4191 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:50.255074807Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"datafile"}
{"msg":"path is \"/data/regions/4570144a-fdc4-4f97-ac49-4899197679e3\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:50.562561831Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"datafile"}
{"msg":"dataset is oxp_8acca6af-d2ba-40aa-a651-5a9c30020d71/crucible/regions/4570144a-fdc4-4f97-ac49-4899197679e3","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:50.571259464Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"datafile"}
{"msg":"snapshot is oxp_8acca6af-d2ba-40aa-a651-5a9c30020d71/crucible/regions/4570144a-fdc4-4f97-ac49-4899197679e3@215b9cb3-2af4-4667-a31f-61646d6a4191","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:50.579119834Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_8acca6af-d2ba-40aa-a651-5a9c30020d71/crucible/regions/4570144a-fdc4-4f97-ac49-4899197679e3\", \"215b9cb3-2af4-4667-a31f-61646d6a4191\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:50.579162135Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:50.587871218Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"uri":"/crucible/0/regions/4570144a-fdc4-4f97-ac49-4899197679e3/snapshots/215b9cb3-2af4-4667-a31f-61646d6a4191/run","method":"POST","req_id":"c5b98574-df8d-4f42-9c27-3abcae683f49","remote_addr":"[fd00:1122:3344:101::b]:53342","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":25310,"response_code":"200"}
{"msg":"region 3d7f04b0-295c-4848-9921-e70761ff24ef state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:55.90405357Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:55.904282776Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"uri":"/crucible/0/regions","method":"POST","req_id":"caa3a3ea-f98c-4833-ae8f-faa88717a96a","remote_addr":"[fd00:1122:3344:101::b]:53342","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":270,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:55.904329727Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_8acca6af-d2ba-40aa-a651-5a9c30020d71/crucible/regions/3d7f04b0-295c-4848-9921-e70761ff24ef","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:55.914586122Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_8acca6af-d2ba-40aa-a651-5a9c30020d71/crucible/regions/3d7f04b0-295c-4848-9921-e70761ff24ef","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:55.914618793Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"3d7f04b0-295c-4848-9921-e70761ff24ef\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/3d7f04b0-295c-4848-9921-e70761ff24ef\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:55.965078532Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"region":"3d7f04b0-295c-4848-9921-e70761ff24ef","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"3d7f04b0-295c-4848-9921-e70761ff24ef\" \"--data\" \"/data/regions/3d7f04b0-295c-4848-9921-e70761ff24ef\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:55.967098246Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"region":"3d7f04b0-295c-4848-9921-e70761ff24ef","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:56.06968104Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"region":"3d7f04b0-295c-4848-9921-e70761ff24ef","component":"worker"}
{"msg":"region 3d7f04b0-295c-4848-9921-e70761ff24ef state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:56.069719981Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:56.069837645Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-3d7f04b0-295c-4848-9921-e70761ff24ef","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:56.070268456Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-3d7f04b0-295c-4848-9921-e70761ff24ef","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:56.082922064Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:56.090318592Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-3d7f04b0-295c-4848-9921-e70761ff24ef","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:56.093451386Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3d7f04b0-295c-4848-9921-e70761ff24ef","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:56.093735974Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:56.093793485Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:56.093838387Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:56.093883188Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:56.097740121Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/4570144a-fdc4-4f97-ac49-4899197679e3","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:56.102242761Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:56.102354174Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:56.102439166Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:56.102448937Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:56.102490608Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-4570144a-fdc4-4f97-ac49-4899197679e3","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:56.102538189Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/4570144a-fdc4-4f97-ac49-4899197679e3","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:56.106396252Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:56.106464324Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:56.106523626Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:56.106584537Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:56.110662746Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:56.112622129Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:13:56.128607686Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"uri":"/crucible/0/regions","method":"POST","req_id":"9f1343f3-dd1e-4397-a00b-0e6b3a65a47e","remote_addr":"[fd00:1122:3344:101::b]:53342","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":52,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:14:21.531011823Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"uri":"/crucible/0/regions/3d7f04b0-295c-4848-9921-e70761ff24ef","method":"GET","req_id":"7563e30c-9354-462f-af38-db2b3d49cf20","remote_addr":"[fd00:1122:3344:101::b]:53342","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":45,"response_code":"200"}
{"msg":"path is \"/data/regions/3d7f04b0-295c-4848-9921-e70761ff24ef\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:14:21.53350478Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"datafile"}
{"msg":"dataset is oxp_8acca6af-d2ba-40aa-a651-5a9c30020d71/crucible/regions/3d7f04b0-295c-4848-9921-e70761ff24ef","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:14:21.546149868Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"datafile"}
{"msg":"region 3d7f04b0-295c-4848-9921-e70761ff24ef state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:14:21.555357334Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:14:21.555546839Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:14:21.55557782Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"uri":"/crucible/0/regions/3d7f04b0-295c-4848-9921-e70761ff24ef","method":"DELETE","req_id":"e132e96c-f5ff-458e-9f86-df4201b1dd17","remote_addr":"[fd00:1122:3344:101::b]:53342","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":22149,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-3d7f04b0-295c-4848-9921-e70761ff24ef (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:14:21.55671098Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:14:21.559134115Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"uri":"/crucible/0/regions/3d7f04b0-295c-4848-9921-e70761ff24ef","method":"GET","req_id":"68b8c9df-a592-4045-9023-dd197d0a455f","remote_addr":"[fd00:1122:3344:101::b]:53342","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":41,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/4570144a-fdc4-4f97-ac49-4899197679e3","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:14:21.562947317Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:14:21.56305027Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:14:21.563134622Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:14:21.563156293Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:14:21.563184923Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-4570144a-fdc4-4f97-ac49-4899197679e3","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:14:21.563233845Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/4570144a-fdc4-4f97-ac49-4899197679e3","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:14:21.563569924Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:14:21.563647016Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:14:21.563698487Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:14:21.563755879Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:14:21.568494636Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:14:21.569944314Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_8acca6af-d2ba-40aa-a651-5a9c30020d71/crucible/regions/3d7f04b0-295c-4848-9921-e70761ff24ef\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:14:21.580875957Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"region":"3d7f04b0-295c-4848-9921-e70761ff24ef","component":"worker"}
{"msg":"region 3d7f04b0-295c-4848-9921-e70761ff24ef state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:14:21.634394178Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-22T03:14:21.691430614Z","hostname":"oxz_crucible_f8523907-78bf-49aa-a0bb-138cdeef407c","pid":22171,"uri":"/crucible/0/regions/3d7f04b0-295c-4848-9921-e70761ff24ef","method":"GET","req_id":"e6a426f4-f902-4189-8d15-d514791bb912","remote_addr":"[fd00:1122:3344:101::b]:53342","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":59,"response_code":"200"}