[ Aug  2 17:23:17 Enabled. ]
[ Aug  2 17:23:17 Rereading configuration. ]
[ Aug  2 17:23:19 Rereading configuration. ]
[ Aug  2 17:23:20 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_90c8a05e-fcad-497a-890c-a9f6819a18b4/crucible -l [fd00:1122:3344:101::12]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_90c8a05e-fcad-497a-890c-a9f6819a18b4/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:23:20.591747545Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744}
{"msg":"listen IP: [fd00:1122:3344:101::12]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:23:20.593352576Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:23:20.593366706Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:23:20.613348888Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:23:20.870903054Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:23:39.774166758Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:59063"}
{"msg":"region 21697905-55f4-4652-af74-cf49c4b3a166 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:23:39.775864821Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:23:39.776231368Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"uri":"/crucible/0/regions","method":"POST","req_id":"9462ae6c-ab5c-4305-9fc0-9e71dbd51bb5","remote_addr":"[fd00:1122:3344:101::a]:59063","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":645,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:23:39.77687238Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_90c8a05e-fcad-497a-890c-a9f6819a18b4/crucible/regions/21697905-55f4-4652-af74-cf49c4b3a166","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:23:39.787269699Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_90c8a05e-fcad-497a-890c-a9f6819a18b4/crucible/regions/21697905-55f4-4652-af74-cf49c4b3a166","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:23:39.78730505Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"21697905-55f4-4652-af74-cf49c4b3a166\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/21697905-55f4-4652-af74-cf49c4b3a166\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:23:39.845774888Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"region":"21697905-55f4-4652-af74-cf49c4b3a166","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"21697905-55f4-4652-af74-cf49c4b3a166\" \"--data\" \"/data/regions/21697905-55f4-4652-af74-cf49c4b3a166\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:23:39.847640724Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"region":"21697905-55f4-4652-af74-cf49c4b3a166","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:23:39.998638202Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"region":"21697905-55f4-4652-af74-cf49c4b3a166","component":"worker"}
{"msg":"region 21697905-55f4-4652-af74-cf49c4b3a166 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:23:39.998686922Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:23:39.998804395Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-21697905-55f4-4652-af74-cf49c4b3a166","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:23:39.999221003Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-21697905-55f4-4652-af74-cf49c4b3a166","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:23:40.003293671Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:23:40.013429355Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-21697905-55f4-4652-af74-cf49c4b3a166","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:23:40.016543734Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/21697905-55f4-4652-af74-cf49c4b3a166","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:23:40.017324599Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:23:40.017405371Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:23:40.017453001Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:23:40.017492692Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:23:40.024430085Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:23:40.030674074Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:23:40.137559278Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"uri":"/crucible/0/regions","method":"POST","req_id":"65a55d33-c751-4a81-8b29-5aaf205ee6e7","remote_addr":"[fd00:1122:3344:101::a]:59063","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":65,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:28:50.492506319Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:41077"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:28:50.492630321Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"uri":"/crucible/0/regions/21697905-55f4-4652-af74-cf49c4b3a166","method":"GET","req_id":"137242a5-a25e-4980-89fb-7d283b29111f","remote_addr":"[fd00:1122:3344:101::a]:41077","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":67,"response_code":"200"}
{"msg":"path is \"/data/regions/21697905-55f4-4652-af74-cf49c4b3a166\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:28:50.497000245Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"component":"datafile"}
{"msg":"dataset is oxp_90c8a05e-fcad-497a-890c-a9f6819a18b4/crucible/regions/21697905-55f4-4652-af74-cf49c4b3a166","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:28:50.505537148Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"component":"datafile"}
{"msg":"snapshot is oxp_90c8a05e-fcad-497a-890c-a9f6819a18b4/crucible/regions/21697905-55f4-4652-af74-cf49c4b3a166@4902fb3a-b3e5-4cfd-8a5d-36861677a655","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:28:50.514176433Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_90c8a05e-fcad-497a-890c-a9f6819a18b4/crucible/regions/21697905-55f4-4652-af74-cf49c4b3a166\", \"4902fb3a-b3e5-4cfd-8a5d-36861677a655\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:28:50.514209364Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:28:50.52552968Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"uri":"/crucible/0/regions/21697905-55f4-4652-af74-cf49c4b3a166/snapshots/4902fb3a-b3e5-4cfd-8a5d-36861677a655","method":"GET","req_id":"29796bb5-79b1-40ff-a5a0-79154f350ac2","remote_addr":"[fd00:1122:3344:101::a]:41077","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":32759,"response_code":"200"}
{"msg":"path is \"/data/regions/21697905-55f4-4652-af74-cf49c4b3a166\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:28:50.525683493Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"component":"datafile"}
{"msg":"dataset is oxp_90c8a05e-fcad-497a-890c-a9f6819a18b4/crucible/regions/21697905-55f4-4652-af74-cf49c4b3a166","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:28:50.532999753Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"component":"datafile"}
{"msg":"snapshot is oxp_90c8a05e-fcad-497a-890c-a9f6819a18b4/crucible/regions/21697905-55f4-4652-af74-cf49c4b3a166@4902fb3a-b3e5-4cfd-8a5d-36861677a655","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:28:50.541173589Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_90c8a05e-fcad-497a-890c-a9f6819a18b4/crucible/regions/21697905-55f4-4652-af74-cf49c4b3a166\", \"4902fb3a-b3e5-4cfd-8a5d-36861677a655\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:28:50.54120069Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 21697905-55f4-4652-af74-cf49c4b3a166-4902fb3a-b3e5-4cfd-8a5d-36861677a655 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:28:50.549967387Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"component":"datafile"}
{"msg":"applying SMF actions for region 21697905-55f4-4652-af74-cf49c4b3a166 running snapshot 4902fb3a-b3e5-4cfd-8a5d-36861677a655 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:28:50.550571329Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:28:50.550585269Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"uri":"/crucible/0/regions/21697905-55f4-4652-af74-cf49c4b3a166/snapshots/4902fb3a-b3e5-4cfd-8a5d-36861677a655/run","method":"POST","req_id":"14d00312-4945-4e46-969f-4be5f887e290","remote_addr":"[fd00:1122:3344:101::a]:41077","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":24918,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-21697905-55f4-4652-af74-cf49c4b3a166-4902fb3a-b3e5-4cfd-8a5d-36861677a655","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:28:50.556917971Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-21697905-55f4-4652-af74-cf49c4b3a166-4902fb3a-b3e5-4cfd-8a5d-36861677a655","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:28:50.569349808Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-21697905-55f4-4652-af74-cf49c4b3a166-4902fb3a-b3e5-4cfd-8a5d-36861677a655","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:28:50.590182937Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/21697905-55f4-4652-af74-cf49c4b3a166/.zfs/snapshot/4902fb3a-b3e5-4cfd-8a5d-36861677a655","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:28:50.590511523Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:28:50.590578204Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:28:50.590628585Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:28:50.590677166Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:28:50.590740387Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:28:50.597767382Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:28:50.606432738Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"component":"worker"}
{"msg":"region 21697905-55f4-4652-af74-cf49c4b3a166 running snapshot 4902fb3a-b3e5-4cfd-8a5d-36861677a655 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-02T17:28:50.606452958Z","hostname":"oxz_crucible_baa56d69-30f4-44fc-974f-238309876fd6","pid":22744,"component":"datafile"}