[ Jul  3 17:22:24 Enabled. ]
[ Jul  3 17:22:24 Rereading configuration. ]
[ Jul  3 17:22:27 Rereading configuration. ]
[ Jul  3 17:22:31 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_041d2335-724e-40fa-b471-4dbb693450ef/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_041d2335-724e-40fa-b471-4dbb693450ef/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:31.438024808Z","hostname":"oxz_crucible_7f70b867-7f64-4875-97f2-657cb79de68e","pid":26633}
{"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:31.441567671Z","hostname":"oxz_crucible_7f70b867-7f64-4875-97f2-657cb79de68e","pid":26633}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:31.441579551Z","hostname":"oxz_crucible_7f70b867-7f64-4875-97f2-657cb79de68e","pid":26633}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:31.456293786Z","hostname":"oxz_crucible_7f70b867-7f64-4875-97f2-657cb79de68e","pid":26633,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:31.69140439Z","hostname":"oxz_crucible_7f70b867-7f64-4875-97f2-657cb79de68e","pid":26633,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.573192249Z","hostname":"oxz_crucible_7f70b867-7f64-4875-97f2-657cb79de68e","pid":26633,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:55839"}
{"msg":"region 6d788602-2524-487d-a73c-632d7d983844 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.600265537Z","hostname":"oxz_crucible_7f70b867-7f64-4875-97f2-657cb79de68e","pid":26633,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.609810717Z","hostname":"oxz_crucible_7f70b867-7f64-4875-97f2-657cb79de68e","pid":26633,"uri":"/crucible/0/regions","method":"POST","req_id":"b788d683-9a5f-4faf-952d-08aeb859a8ff","remote_addr":"[fd00:1122:3344:101::a]:55839","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":21789,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.61219595Z","hostname":"oxz_crucible_7f70b867-7f64-4875-97f2-657cb79de68e","pid":26633,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_041d2335-724e-40fa-b471-4dbb693450ef/crucible/regions/6d788602-2524-487d-a73c-632d7d983844","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.625535719Z","hostname":"oxz_crucible_7f70b867-7f64-4875-97f2-657cb79de68e","pid":26633,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_041d2335-724e-40fa-b471-4dbb693450ef/crucible/regions/6d788602-2524-487d-a73c-632d7d983844","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.625567589Z","hostname":"oxz_crucible_7f70b867-7f64-4875-97f2-657cb79de68e","pid":26633,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"6d788602-2524-487d-a73c-632d7d983844\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/6d788602-2524-487d-a73c-632d7d983844\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.67486781Z","hostname":"oxz_crucible_7f70b867-7f64-4875-97f2-657cb79de68e","pid":26633,"region":"6d788602-2524-487d-a73c-632d7d983844","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"6d788602-2524-487d-a73c-632d7d983844\" \"--data\" \"/data/regions/6d788602-2524-487d-a73c-632d7d983844\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.677078258Z","hostname":"oxz_crucible_7f70b867-7f64-4875-97f2-657cb79de68e","pid":26633,"region":"6d788602-2524-487d-a73c-632d7d983844","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.923919829Z","hostname":"oxz_crucible_7f70b867-7f64-4875-97f2-657cb79de68e","pid":26633,"uri":"/crucible/0/regions","method":"POST","req_id":"352f5cb1-8ba3-4842-84ce-ed2baad7af22","remote_addr":"[fd00:1122:3344:101::a]:55839","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":79,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.957212661Z","hostname":"oxz_crucible_7f70b867-7f64-4875-97f2-657cb79de68e","pid":26633,"region":"6d788602-2524-487d-a73c-632d7d983844","component":"worker"}
{"msg":"region 6d788602-2524-487d-a73c-632d7d983844 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.957238981Z","hostname":"oxz_crucible_7f70b867-7f64-4875-97f2-657cb79de68e","pid":26633,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.957375095Z","hostname":"oxz_crucible_7f70b867-7f64-4875-97f2-657cb79de68e","pid":26633,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-6d788602-2524-487d-a73c-632d7d983844","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.957862607Z","hostname":"oxz_crucible_7f70b867-7f64-4875-97f2-657cb79de68e","pid":26633,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-6d788602-2524-487d-a73c-632d7d983844","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.965112267Z","hostname":"oxz_crucible_7f70b867-7f64-4875-97f2-657cb79de68e","pid":26633,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.98203215Z","hostname":"oxz_crucible_7f70b867-7f64-4875-97f2-657cb79de68e","pid":26633,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-6d788602-2524-487d-a73c-632d7d983844","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.984459204Z","hostname":"oxz_crucible_7f70b867-7f64-4875-97f2-657cb79de68e","pid":26633,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/6d788602-2524-487d-a73c-632d7d983844","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.985005348Z","hostname":"oxz_crucible_7f70b867-7f64-4875-97f2-657cb79de68e","pid":26633,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.987405071Z","hostname":"oxz_crucible_7f70b867-7f64-4875-97f2-657cb79de68e","pid":26633,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.987471922Z","hostname":"oxz_crucible_7f70b867-7f64-4875-97f2-657cb79de68e","pid":26633,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.987523564Z","hostname":"oxz_crucible_7f70b867-7f64-4875-97f2-657cb79de68e","pid":26633,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.996187701Z","hostname":"oxz_crucible_7f70b867-7f64-4875-97f2-657cb79de68e","pid":26633,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:12.002710441Z","hostname":"oxz_crucible_7f70b867-7f64-4875-97f2-657cb79de68e","pid":26633,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:12.49317796Z","hostname":"oxz_crucible_7f70b867-7f64-4875-97f2-657cb79de68e","pid":26633,"uri":"/crucible/0/regions","method":"POST","req_id":"a2568c99-4645-4735-b700-ef385fe071a9","remote_addr":"[fd00:1122:3344:101::a]:55839","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":85,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:36.540207524Z","hostname":"oxz_crucible_7f70b867-7f64-4875-97f2-657cb79de68e","pid":26633,"uri":"/crucible/0/regions/6d788602-2524-487d-a73c-632d7d983844","method":"GET","req_id":"8f578948-43e5-402d-8756-371bb6874cb8","remote_addr":"[fd00:1122:3344:101::a]:55839","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":590,"response_code":"200"}
{"msg":"path is \"/data/regions/6d788602-2524-487d-a73c-632d7d983844\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:36.542894694Z","hostname":"oxz_crucible_7f70b867-7f64-4875-97f2-657cb79de68e","pid":26633,"component":"datafile"}
{"msg":"dataset is oxp_041d2335-724e-40fa-b471-4dbb693450ef/crucible/regions/6d788602-2524-487d-a73c-632d7d983844","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:36.55306603Z","hostname":"oxz_crucible_7f70b867-7f64-4875-97f2-657cb79de68e","pid":26633,"component":"datafile"}
{"msg":"region 6d788602-2524-487d-a73c-632d7d983844 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:36.560554437Z","hostname":"oxz_crucible_7f70b867-7f64-4875-97f2-657cb79de68e","pid":26633,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:36.560722111Z","hostname":"oxz_crucible_7f70b867-7f64-4875-97f2-657cb79de68e","pid":26633,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:36.560792843Z","hostname":"oxz_crucible_7f70b867-7f64-4875-97f2-657cb79de68e","pid":26633,"uri":"/crucible/0/regions/6d788602-2524-487d-a73c-632d7d983844","method":"DELETE","req_id":"0b880064-23eb-4417-968a-5b3f391e8e6d","remote_addr":"[fd00:1122:3344:101::a]:55839","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":17969,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-6d788602-2524-487d-a73c-632d7d983844 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:36.561929673Z","hostname":"oxz_crucible_7f70b867-7f64-4875-97f2-657cb79de68e","pid":26633,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:36.56335565Z","hostname":"oxz_crucible_7f70b867-7f64-4875-97f2-657cb79de68e","pid":26633,"uri":"/crucible/0/regions/6d788602-2524-487d-a73c-632d7d983844","method":"GET","req_id":"3ceab633-3b99-4cb2-8694-7a456281714b","remote_addr":"[fd00:1122:3344:101::a]:55839","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":26,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:36.56567488Z","hostname":"oxz_crucible_7f70b867-7f64-4875-97f2-657cb79de68e","pid":26633,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_041d2335-724e-40fa-b471-4dbb693450ef/crucible/regions/6d788602-2524-487d-a73c-632d7d983844\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:36.575179639Z","hostname":"oxz_crucible_7f70b867-7f64-4875-97f2-657cb79de68e","pid":26633,"region":"6d788602-2524-487d-a73c-632d7d983844","component":"worker"}
{"msg":"region 6d788602-2524-487d-a73c-632d7d983844 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:36.622649302Z","hostname":"oxz_crucible_7f70b867-7f64-4875-97f2-657cb79de68e","pid":26633,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:36.655155883Z","hostname":"oxz_crucible_7f70b867-7f64-4875-97f2-657cb79de68e","pid":26633,"uri":"/crucible/0/regions/6d788602-2524-487d-a73c-632d7d983844","method":"GET","req_id":"e77d07ce-623d-4476-9f5f-ba5bd9d10a9e","remote_addr":"[fd00:1122:3344:101::a]:55839","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":66,"response_code":"200"}