[ Jul  1 10:27:09 Enabled. ]
[ Jul  1 10:27:09 Rereading configuration. ]
[ Jul  1 10:27:11 Rereading configuration. ]
[ Jul  1 10:27:14 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_8c78f299-6347-4250-8620-7e094314528f/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_8c78f299-6347-4250-8620-7e094314528f/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:27:14.24081348Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130}
{"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:27:14.244476472Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:27:14.244488503Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:27:14.26558101Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:27:14.378641429Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:07.078706731Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:60617"}
{"msg":"region 0505bdc7-0034-4887-bfdf-f58d867c6247 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:07.114026974Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:07.118944671Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"uri":"/crucible/0/regions","method":"POST","req_id":"71198fe4-c53e-41a3-8da6-c0fe0a12c8ff","remote_addr":"[fd00:1122:3344:101::c]:60617","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":24443,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:07.121445831Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_8c78f299-6347-4250-8620-7e094314528f/crucible/regions/0505bdc7-0034-4887-bfdf-f58d867c6247","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:07.131371778Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_8c78f299-6347-4250-8620-7e094314528f/crucible/regions/0505bdc7-0034-4887-bfdf-f58d867c6247","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:07.131403698Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"0505bdc7-0034-4887-bfdf-f58d867c6247\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/0505bdc7-0034-4887-bfdf-f58d867c6247\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:07.192580542Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"region":"0505bdc7-0034-4887-bfdf-f58d867c6247","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"0505bdc7-0034-4887-bfdf-f58d867c6247\" \"--data\" \"/data/regions/0505bdc7-0034-4887-bfdf-f58d867c6247\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:07.192644674Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"region":"0505bdc7-0034-4887-bfdf-f58d867c6247","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:07.385414093Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"uri":"/crucible/0/regions","method":"POST","req_id":"73d8e8b4-733c-4ae6-921a-557e4fa30fe3","remote_addr":"[fd00:1122:3344:101::c]:60617","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":77,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:07.51776831Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"region":"0505bdc7-0034-4887-bfdf-f58d867c6247","component":"worker"}
{"msg":"region 0505bdc7-0034-4887-bfdf-f58d867c6247 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:07.517818431Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:07.517938524Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-0505bdc7-0034-4887-bfdf-f58d867c6247","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:07.518327735Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-0505bdc7-0034-4887-bfdf-f58d867c6247","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:07.523759786Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:07.540950685Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-0505bdc7-0034-4887-bfdf-f58d867c6247","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:07.544158375Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/0505bdc7-0034-4887-bfdf-f58d867c6247","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:07.544881045Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:07.544940916Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:07.544980918Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:07.545017629Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:07.549304828Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:07.553155635Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:07.931782851Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"uri":"/crucible/0/regions","method":"POST","req_id":"fb43046f-a94a-432a-9d70-9aa0a4b06dd8","remote_addr":"[fd00:1122:3344:101::c]:60617","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":51,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:08.237710772Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"uri":"/crucible/0/regions/0505bdc7-0034-4887-bfdf-f58d867c6247","method":"GET","req_id":"8df66e41-1209-4c40-8e3e-aa29fac68c3e","remote_addr":"[fd00:1122:3344:101::c]:60617","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":111,"response_code":"200"}
{"msg":"path is \"/data/regions/0505bdc7-0034-4887-bfdf-f58d867c6247\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:08.240500949Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"component":"datafile"}
{"msg":"dataset is oxp_8c78f299-6347-4250-8620-7e094314528f/crucible/regions/0505bdc7-0034-4887-bfdf-f58d867c6247","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:08.251786574Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"component":"datafile"}
{"msg":"region 0505bdc7-0034-4887-bfdf-f58d867c6247 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:08.262361108Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:08.262542193Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:08.262618176Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"uri":"/crucible/0/regions/0505bdc7-0034-4887-bfdf-f58d867c6247","method":"DELETE","req_id":"1fbfbd7f-b26a-4ae6-8a44-d3a643f00b65","remote_addr":"[fd00:1122:3344:101::c]:60617","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":22150,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-0505bdc7-0034-4887-bfdf-f58d867c6247 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:08.263520301Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:08.265896787Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"uri":"/crucible/0/regions/0505bdc7-0034-4887-bfdf-f58d867c6247","method":"GET","req_id":"a921f70c-8680-4ea9-899b-4213f6062331","remote_addr":"[fd00:1122:3344:101::c]:60617","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":29,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:08.267384808Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_8c78f299-6347-4250-8620-7e094314528f/crucible/regions/0505bdc7-0034-4887-bfdf-f58d867c6247\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:08.27784884Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"region":"0505bdc7-0034-4887-bfdf-f58d867c6247","component":"worker"}
{"msg":"region 0505bdc7-0034-4887-bfdf-f58d867c6247 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:08.342919252Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:08.393342077Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"uri":"/crucible/0/regions/0505bdc7-0034-4887-bfdf-f58d867c6247","method":"GET","req_id":"96728411-cbed-4906-a0f0-c29e3356096a","remote_addr":"[fd00:1122:3344:101::c]:60617","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":64,"response_code":"200"}
{"msg":"region 86f114ad-7b10-43c4-bb35-3d544d152764 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:10.133014611Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:10.133229697Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:10.133284769Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"uri":"/crucible/0/regions","method":"POST","req_id":"9d148b6e-f404-4e99-894a-815d54c28b85","remote_addr":"[fd00:1122:3344:101::c]:60617","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":298,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_8c78f299-6347-4250-8620-7e094314528f/crucible/regions/86f114ad-7b10-43c4-bb35-3d544d152764","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:10.143892824Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_8c78f299-6347-4250-8620-7e094314528f/crucible/regions/86f114ad-7b10-43c4-bb35-3d544d152764","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:10.143918415Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"86f114ad-7b10-43c4-bb35-3d544d152764\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/86f114ad-7b10-43c4-bb35-3d544d152764\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:10.201796087Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"region":"86f114ad-7b10-43c4-bb35-3d544d152764","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"86f114ad-7b10-43c4-bb35-3d544d152764\" \"--data\" \"/data/regions/86f114ad-7b10-43c4-bb35-3d544d152764\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:10.203733901Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"region":"86f114ad-7b10-43c4-bb35-3d544d152764","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:10.272743983Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"uri":"/crucible/0/regions","method":"POST","req_id":"6d386a65-1479-4127-881f-d3babafe3227","remote_addr":"[fd00:1122:3344:101::c]:60617","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":83,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:10.284723926Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"region":"86f114ad-7b10-43c4-bb35-3d544d152764","component":"worker"}
{"msg":"region 86f114ad-7b10-43c4-bb35-3d544d152764 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:10.284744607Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:10.284864781Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-86f114ad-7b10-43c4-bb35-3d544d152764","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:10.286087055Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-86f114ad-7b10-43c4-bb35-3d544d152764","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:10.289111109Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:10.296468774Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-86f114ad-7b10-43c4-bb35-3d544d152764","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:10.299527939Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/86f114ad-7b10-43c4-bb35-3d544d152764","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:10.299839258Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:10.299883129Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:10.29993837Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:10.299980171Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:10.303732976Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:10.308120128Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:28:10.915923377Z","hostname":"oxz_crucible_7c085edc-40e8-4adb-8eb8-677917336515","pid":21130,"uri":"/crucible/0/regions","method":"POST","req_id":"e957fcd3-4182-46e0-ac1d-af18a7abc0cf","remote_addr":"[fd00:1122:3344:101::c]:60617","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":50,"response_code":"200"}