[ Jul 10 20:02:45 Enabled. ]
[ Jul 10 20:02:45 Rereading configuration. ]
[ Jul 10 20:02:46 Rereading configuration. ]
[ Jul 10 20:02:48 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_c50a0442-8d96-40c6-91eb-e3a7f4bff366/crucible -l [fd00:1122:3344:101::17]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_c50a0442-8d96-40c6-91eb-e3a7f4bff366/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:02:48.782962411Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387}
{"msg":"listen IP: [fd00:1122:3344:101::17]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:02:48.78761627Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:02:48.78763569Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:02:48.808313032Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:02:48.877205862Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.536750564Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:61808"}
{"msg":"region e51d5658-8ff1-40d8-a072-dafdc1aaa10e state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.556052124Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.561258437Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"uri":"/crucible/0/regions","method":"POST","req_id":"5d077ed3-4cd2-4b52-87dc-db4940f5ea8a","remote_addr":"[fd00:1122:3344:101::c]:61808","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":13523,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.561281847Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_c50a0442-8d96-40c6-91eb-e3a7f4bff366/crucible/regions/e51d5658-8ff1-40d8-a072-dafdc1aaa10e","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.575309084Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_c50a0442-8d96-40c6-91eb-e3a7f4bff366/crucible/regions/e51d5658-8ff1-40d8-a072-dafdc1aaa10e","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.575347724Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"e51d5658-8ff1-40d8-a072-dafdc1aaa10e\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/e51d5658-8ff1-40d8-a072-dafdc1aaa10e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.667634399Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"region":"e51d5658-8ff1-40d8-a072-dafdc1aaa10e","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"e51d5658-8ff1-40d8-a072-dafdc1aaa10e\" \"--data\" \"/data/regions/e51d5658-8ff1-40d8-a072-dafdc1aaa10e\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.669916098Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"region":"e51d5658-8ff1-40d8-a072-dafdc1aaa10e","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.875507281Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"region":"e51d5658-8ff1-40d8-a072-dafdc1aaa10e","component":"worker"}
{"msg":"region e51d5658-8ff1-40d8-a072-dafdc1aaa10e state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.875577422Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.876361458Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-e51d5658-8ff1-40d8-a072-dafdc1aaa10e","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.876824542Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-e51d5658-8ff1-40d8-a072-dafdc1aaa10e","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.896032471Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.909630774Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"uri":"/crucible/0/regions","method":"POST","req_id":"2651c17f-663c-432a-b251-5d8f956cd1ce","remote_addr":"[fd00:1122:3344:101::c]:61808","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":82,"response_code":"200"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.937256403Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e51d5658-8ff1-40d8-a072-dafdc1aaa10e","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.95867493Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e51d5658-8ff1-40d8-a072-dafdc1aaa10e","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.959507807Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.9634571Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.96352542Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.963565481Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:14.006489847Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:14.025160651Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:15.550486782Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"uri":"/crucible/0/regions/e51d5658-8ff1-40d8-a072-dafdc1aaa10e","method":"GET","req_id":"e5786fde-feb3-4156-9ea7-174f05041fdf","remote_addr":"[fd00:1122:3344:101::c]:61808","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":110,"response_code":"200"}
{"msg":"path is \"/data/regions/e51d5658-8ff1-40d8-a072-dafdc1aaa10e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:15.553719629Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"component":"datafile"}
{"msg":"dataset is oxp_c50a0442-8d96-40c6-91eb-e3a7f4bff366/crucible/regions/e51d5658-8ff1-40d8-a072-dafdc1aaa10e","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:15.563673161Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"component":"datafile"}
{"msg":"region e51d5658-8ff1-40d8-a072-dafdc1aaa10e state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:15.572993889Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:15.57315564Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:15.57321873Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"uri":"/crucible/0/regions/e51d5658-8ff1-40d8-a072-dafdc1aaa10e","method":"DELETE","req_id":"c542aadc-1b47-47fe-8a1d-c2134cb8ff89","remote_addr":"[fd00:1122:3344:101::c]:61808","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":19540,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-e51d5658-8ff1-40d8-a072-dafdc1aaa10e (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:15.574302589Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:15.575899713Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"uri":"/crucible/0/regions/e51d5658-8ff1-40d8-a072-dafdc1aaa10e","method":"GET","req_id":"e0aebe42-8cde-46b9-83b2-6a78fc8637a8","remote_addr":"[fd00:1122:3344:101::c]:61808","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":577,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:15.598026576Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_c50a0442-8d96-40c6-91eb-e3a7f4bff366/crucible/regions/e51d5658-8ff1-40d8-a072-dafdc1aaa10e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:15.607746657Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"region":"e51d5658-8ff1-40d8-a072-dafdc1aaa10e","component":"worker"}
{"msg":"region e51d5658-8ff1-40d8-a072-dafdc1aaa10e state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:15.683947068Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:15.690951256Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"uri":"/crucible/0/regions/e51d5658-8ff1-40d8-a072-dafdc1aaa10e","method":"GET","req_id":"a910b151-d01e-419e-990b-37ac2bf6a314","remote_addr":"[fd00:1122:3344:101::c]:61808","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":80,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.298100805Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:63966"}
{"msg":"region 0e99b425-d51d-4f9e-be7e-42fd3639810e state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.298241896Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.301050089Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"uri":"/crucible/0/regions","method":"POST","req_id":"2d125321-7449-46f0-8721-2f536e2816c1","remote_addr":"[fd00:1122:3344:101::a]:63966","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":2904,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.30107302Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_c50a0442-8d96-40c6-91eb-e3a7f4bff366/crucible/regions/0e99b425-d51d-4f9e-be7e-42fd3639810e","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.311630067Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_c50a0442-8d96-40c6-91eb-e3a7f4bff366/crucible/regions/0e99b425-d51d-4f9e-be7e-42fd3639810e","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.311655367Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"0e99b425-d51d-4f9e-be7e-42fd3639810e\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/0e99b425-d51d-4f9e-be7e-42fd3639810e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.362183316Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"region":"0e99b425-d51d-4f9e-be7e-42fd3639810e","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"0e99b425-d51d-4f9e-be7e-42fd3639810e\" \"--data\" \"/data/regions/0e99b425-d51d-4f9e-be7e-42fd3639810e\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.364176083Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"region":"0e99b425-d51d-4f9e-be7e-42fd3639810e","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.42537303Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"region":"0e99b425-d51d-4f9e-be7e-42fd3639810e","component":"worker"}
{"msg":"region 0e99b425-d51d-4f9e-be7e-42fd3639810e state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.4254076Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.425498301Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-0e99b425-d51d-4f9e-be7e-42fd3639810e","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.426862342Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-0e99b425-d51d-4f9e-be7e-42fd3639810e","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.431913984Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.446483745Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-0e99b425-d51d-4f9e-be7e-42fd3639810e","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.449033486Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/0e99b425-d51d-4f9e-be7e-42fd3639810e","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.449333278Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.449395939Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.449438679Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.44948163Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.452700066Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.457121543Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:09:21.509635648Z","hostname":"oxz_crucible_efbe23b2-247d-4166-ad08-1f54597d3f72","pid":25387,"uri":"/crucible/0/regions","method":"POST","req_id":"a54f5ab9-d3d1-4d9c-8840-54f6e1ae30db","remote_addr":"[fd00:1122:3344:101::a]:63966","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":643,"response_code":"200"}