[ Dec 21 01:16:05 Enabled. ]
[ Dec 21 01:16:05 Rereading configuration. ]
[ Dec 21 01:16:09 Rereading configuration. ]
[ Dec 21 01:16:13 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_41cc83ed-adbe-492f-ae31-7074b31692b0/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_41cc83ed-adbe-492f-ae31-7074b31692b0/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:16:13.483944436Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095}
{"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:16:13.487065448Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:16:13.487075018Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:16:13.500961932Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:16:13.59648665Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.55680073Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:42021"}
{"msg":"region 51a8d9e9-4f1b-4df5-85d0-b8c373eb413a state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.580990777Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.58593124Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.590785252Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"uri":"/crucible/0/regions","method":"POST","req_id":"6bde8512-3ef5-471e-a7ea-d9c5adde4bae","remote_addr":"[fd00:1122:3344:101::a]:42021","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":13776,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_41cc83ed-adbe-492f-ae31-7074b31692b0/crucible/regions/51a8d9e9-4f1b-4df5-85d0-b8c373eb413a","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.596632901Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_41cc83ed-adbe-492f-ae31-7074b31692b0/crucible/regions/51a8d9e9-4f1b-4df5-85d0-b8c373eb413a","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.596658031Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"51a8d9e9-4f1b-4df5-85d0-b8c373eb413a\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/51a8d9e9-4f1b-4df5-85d0-b8c373eb413a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.653727892Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"region":"51a8d9e9-4f1b-4df5-85d0-b8c373eb413a","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"51a8d9e9-4f1b-4df5-85d0-b8c373eb413a\" \"--data\" \"/data/regions/51a8d9e9-4f1b-4df5-85d0-b8c373eb413a\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.655664815Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"region":"51a8d9e9-4f1b-4df5-85d0-b8c373eb413a","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.785189095Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"region":"51a8d9e9-4f1b-4df5-85d0-b8c373eb413a","component":"worker"}
{"msg":"region 51a8d9e9-4f1b-4df5-85d0-b8c373eb413a state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.785250116Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.785319197Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-51a8d9e9-4f1b-4df5-85d0-b8c373eb413a","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.785706014Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-51a8d9e9-4f1b-4df5-85d0-b8c373eb413a","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.789534299Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.800947421Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-51a8d9e9-4f1b-4df5-85d0-b8c373eb413a","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.804286847Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/51a8d9e9-4f1b-4df5-85d0-b8c373eb413a","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.804984089Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.80504895Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.805096941Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.805137091Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.809050517Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.812890632Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.914694936Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"uri":"/crucible/0/regions","method":"POST","req_id":"83fc9b15-a9a4-4a9a-8146-73eb7c9feb17","remote_addr":"[fd00:1122:3344:101::a]:42021","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":70,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:36.560760906Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:45755"}
{"msg":"region 05a8143b-d69e-450e-a29f-20ae9edb35da state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:36.560900089Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:36.56158857Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"uri":"/crucible/0/regions","method":"POST","req_id":"ce561d13-80ae-40ef-9929-4d1e4e15bf20","remote_addr":"[fd00:1122:3344:101::a]:45755","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":716,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:36.561607491Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_41cc83ed-adbe-492f-ae31-7074b31692b0/crucible/regions/05a8143b-d69e-450e-a29f-20ae9edb35da","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:36.573757015Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_41cc83ed-adbe-492f-ae31-7074b31692b0/crucible/regions/05a8143b-d69e-450e-a29f-20ae9edb35da","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:36.573791456Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"05a8143b-d69e-450e-a29f-20ae9edb35da\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/05a8143b-d69e-450e-a29f-20ae9edb35da\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:36.643979478Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"region":"05a8143b-d69e-450e-a29f-20ae9edb35da","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"05a8143b-d69e-450e-a29f-20ae9edb35da\" \"--data\" \"/data/regions/05a8143b-d69e-450e-a29f-20ae9edb35da\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:36.644035979Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"region":"05a8143b-d69e-450e-a29f-20ae9edb35da","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:36.718059925Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"region":"05a8143b-d69e-450e-a29f-20ae9edb35da","component":"worker"}
{"msg":"region 05a8143b-d69e-450e-a29f-20ae9edb35da state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:36.718103806Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:36.718214717Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-05a8143b-d69e-450e-a29f-20ae9edb35da","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:36.718683295Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-05a8143b-d69e-450e-a29f-20ae9edb35da","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:36.724751938Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:36.73857989Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-05a8143b-d69e-450e-a29f-20ae9edb35da","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:36.741851115Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/05a8143b-d69e-450e-a29f-20ae9edb35da","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:36.74213486Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:36.742194051Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:36.742238442Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:36.742279102Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:36.746475633Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:36.754793373Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:36.864650923Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"uri":"/crucible/0/regions","method":"POST","req_id":"3471fa30-ae8b-4a01-b43a-4d68cbc148cb","remote_addr":"[fd00:1122:3344:101::a]:45755","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":71,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:38.433247941Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"uri":"/crucible/0/regions/51a8d9e9-4f1b-4df5-85d0-b8c373eb413a","method":"GET","req_id":"614729f4-7d33-4822-beb7-7afa78db83c0","remote_addr":"[fd00:1122:3344:101::a]:45755","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":2753,"response_code":"200"}
{"msg":"path is \"/data/regions/51a8d9e9-4f1b-4df5-85d0-b8c373eb413a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:38.436855022Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"datafile"}
{"msg":"dataset is oxp_41cc83ed-adbe-492f-ae31-7074b31692b0/crucible/regions/51a8d9e9-4f1b-4df5-85d0-b8c373eb413a","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:38.4456604Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"datafile"}
{"msg":"snapshot is oxp_41cc83ed-adbe-492f-ae31-7074b31692b0/crucible/regions/51a8d9e9-4f1b-4df5-85d0-b8c373eb413a@f46300ad-91ae-40ca-bb40-7dba2e257bcc","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:38.454028751Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_41cc83ed-adbe-492f-ae31-7074b31692b0/crucible/regions/51a8d9e9-4f1b-4df5-85d0-b8c373eb413a\", \"f46300ad-91ae-40ca-bb40-7dba2e257bcc\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:38.454065202Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:38.464409706Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"uri":"/crucible/0/regions/51a8d9e9-4f1b-4df5-85d0-b8c373eb413a/snapshots/f46300ad-91ae-40ca-bb40-7dba2e257bcc","method":"GET","req_id":"44077e97-8542-4335-ab4a-8145cb4c9ad3","remote_addr":"[fd00:1122:3344:101::a]:45755","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":27569,"response_code":"200"}
{"msg":"path is \"/data/regions/51a8d9e9-4f1b-4df5-85d0-b8c373eb413a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:38.467740122Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"datafile"}
{"msg":"dataset is oxp_41cc83ed-adbe-492f-ae31-7074b31692b0/crucible/regions/51a8d9e9-4f1b-4df5-85d0-b8c373eb413a","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:38.475291939Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"datafile"}
{"msg":"snapshot is oxp_41cc83ed-adbe-492f-ae31-7074b31692b0/crucible/regions/51a8d9e9-4f1b-4df5-85d0-b8c373eb413a@f46300ad-91ae-40ca-bb40-7dba2e257bcc","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:38.483887914Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_41cc83ed-adbe-492f-ae31-7074b31692b0/crucible/regions/51a8d9e9-4f1b-4df5-85d0-b8c373eb413a\", \"f46300ad-91ae-40ca-bb40-7dba2e257bcc\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:38.483914554Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 51a8d9e9-4f1b-4df5-85d0-b8c373eb413a-f46300ad-91ae-40ca-bb40-7dba2e257bcc state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:38.492314146Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"datafile"}
{"msg":"applying SMF actions for region 51a8d9e9-4f1b-4df5-85d0-b8c373eb413a running snapshot f46300ad-91ae-40ca-bb40-7dba2e257bcc (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:38.492522189Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:38.49255173Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"uri":"/crucible/0/regions/51a8d9e9-4f1b-4df5-85d0-b8c373eb413a/snapshots/f46300ad-91ae-40ca-bb40-7dba2e257bcc/run","method":"POST","req_id":"74a75d11-903b-41bc-b197-cec334f95ae2","remote_addr":"[fd00:1122:3344:101::a]:45755","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":24847,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-51a8d9e9-4f1b-4df5-85d0-b8c373eb413a-f46300ad-91ae-40ca-bb40-7dba2e257bcc","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:38.494752087Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-51a8d9e9-4f1b-4df5-85d0-b8c373eb413a-f46300ad-91ae-40ca-bb40-7dba2e257bcc","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:38.505513508Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-51a8d9e9-4f1b-4df5-85d0-b8c373eb413a-f46300ad-91ae-40ca-bb40-7dba2e257bcc","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:38.516289689Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/51a8d9e9-4f1b-4df5-85d0-b8c373eb413a/.zfs/snapshot/f46300ad-91ae-40ca-bb40-7dba2e257bcc","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:38.516629035Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:38.516681666Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:38.516724377Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:38.516762667Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:38.516812918Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:38.52051492Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:38.524824263Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"region 51a8d9e9-4f1b-4df5-85d0-b8c373eb413a running snapshot f46300ad-91ae-40ca-bb40-7dba2e257bcc state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:38.524838293Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"datafile"}
{"msg":"path is \"/data/regions/51a8d9e9-4f1b-4df5-85d0-b8c373eb413a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:38.832069756Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"datafile"}
{"msg":"dataset is oxp_41cc83ed-adbe-492f-ae31-7074b31692b0/crucible/regions/51a8d9e9-4f1b-4df5-85d0-b8c373eb413a","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:38.841096998Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"datafile"}
{"msg":"snapshot is oxp_41cc83ed-adbe-492f-ae31-7074b31692b0/crucible/regions/51a8d9e9-4f1b-4df5-85d0-b8c373eb413a@f46300ad-91ae-40ca-bb40-7dba2e257bcc","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:38.849782554Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_41cc83ed-adbe-492f-ae31-7074b31692b0/crucible/regions/51a8d9e9-4f1b-4df5-85d0-b8c373eb413a\", \"f46300ad-91ae-40ca-bb40-7dba2e257bcc\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:38.849809565Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:38.857709478Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"uri":"/crucible/0/regions/51a8d9e9-4f1b-4df5-85d0-b8c373eb413a/snapshots/f46300ad-91ae-40ca-bb40-7dba2e257bcc/run","method":"POST","req_id":"3bbc6768-45a2-4bfa-81ac-8c703d3add31","remote_addr":"[fd00:1122:3344:101::a]:45755","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":25663,"response_code":"200"}
{"msg":"region 0039193e-dfb0-42cf-a035-390be3bcb0db state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:42.299445793Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:42.299662636Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"uri":"/crucible/0/regions","method":"POST","req_id":"63601f5c-c418-4281-8095-47a5ee399a61","remote_addr":"[fd00:1122:3344:101::a]:45755","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":277,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:42.299699687Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_41cc83ed-adbe-492f-ae31-7074b31692b0/crucible/regions/0039193e-dfb0-42cf-a035-390be3bcb0db","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:42.30939145Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_41cc83ed-adbe-492f-ae31-7074b31692b0/crucible/regions/0039193e-dfb0-42cf-a035-390be3bcb0db","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:42.30941843Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"0039193e-dfb0-42cf-a035-390be3bcb0db\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/0039193e-dfb0-42cf-a035-390be3bcb0db\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:42.385485221Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"region":"0039193e-dfb0-42cf-a035-390be3bcb0db","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"0039193e-dfb0-42cf-a035-390be3bcb0db\" \"--data\" \"/data/regions/0039193e-dfb0-42cf-a035-390be3bcb0db\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:42.385562912Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"region":"0039193e-dfb0-42cf-a035-390be3bcb0db","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:42.485836291Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"region":"0039193e-dfb0-42cf-a035-390be3bcb0db","component":"worker"}
{"msg":"region 0039193e-dfb0-42cf-a035-390be3bcb0db state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:42.485906572Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:42.485965123Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-0039193e-dfb0-42cf-a035-390be3bcb0db","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:42.486502022Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-0039193e-dfb0-42cf-a035-390be3bcb0db","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:42.489681125Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:42.498110927Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-0039193e-dfb0-42cf-a035-390be3bcb0db","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:42.501499104Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/0039193e-dfb0-42cf-a035-390be3bcb0db","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:42.501751839Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19003","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:42.501812389Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:42.50185706Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:42.501904231Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:42.505726075Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:42.522253824Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:42.523254571Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"uri":"/crucible/0/regions","method":"POST","req_id":"70ce3754-09f0-48dd-be2a-c13f08e49a95","remote_addr":"[fd00:1122:3344:101::a]:45755","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:28:07.625042004Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"uri":"/crucible/0/regions/0039193e-dfb0-42cf-a035-390be3bcb0db","method":"GET","req_id":"db58f650-557b-4ef5-9ea6-918db3be8b67","remote_addr":"[fd00:1122:3344:101::a]:45755","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":63,"response_code":"200"}
{"msg":"path is \"/data/regions/0039193e-dfb0-42cf-a035-390be3bcb0db\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:28:07.628767126Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"datafile"}
{"msg":"dataset is oxp_41cc83ed-adbe-492f-ae31-7074b31692b0/crucible/regions/0039193e-dfb0-42cf-a035-390be3bcb0db","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:28:07.641113594Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"datafile"}
{"msg":"region 0039193e-dfb0-42cf-a035-390be3bcb0db state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:28:07.652310523Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:28:07.652530366Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:28:07.652603838Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"uri":"/crucible/0/regions/0039193e-dfb0-42cf-a035-390be3bcb0db","method":"DELETE","req_id":"9e2a5e84-b67d-4358-a964-425e1f6e113d","remote_addr":"[fd00:1122:3344:101::a]:45755","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":23866,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:28:07.655519897Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"uri":"/crucible/0/regions/0039193e-dfb0-42cf-a035-390be3bcb0db","method":"GET","req_id":"f5e6938d-0af9-4113-9900-76a9ab0aba8d","remote_addr":"[fd00:1122:3344:101::a]:45755","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":25,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-0039193e-dfb0-42cf-a035-390be3bcb0db (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:28:07.65629583Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:28:07.663977119Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_41cc83ed-adbe-492f-ae31-7074b31692b0/crucible/regions/0039193e-dfb0-42cf-a035-390be3bcb0db\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:28:07.676192465Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"region":"0039193e-dfb0-42cf-a035-390be3bcb0db","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:28:07.716086307Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"uri":"/crucible/0/regions/0039193e-dfb0-42cf-a035-390be3bcb0db","method":"GET","req_id":"e6bdd41b-d304-4da2-9dcb-1a37516cfdf9","remote_addr":"[fd00:1122:3344:101::a]:45755","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":744,"response_code":"200"}
{"msg":"region 0039193e-dfb0-42cf-a035-390be3bcb0db state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:28:07.736973568Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:28:07.85584676Z","hostname":"oxz_crucible_698a7a9e-913f-4d1d-8acf-e625ceef34b1","pid":25095,"uri":"/crucible/0/regions/0039193e-dfb0-42cf-a035-390be3bcb0db","method":"GET","req_id":"fa54898b-b7c8-416f-8238-b186a8f0f141","remote_addr":"[fd00:1122:3344:101::a]:45755","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":31,"response_code":"200"}