[ Mar 26 19:47:53 Enabled. ]
[ Mar 26 19:47:53 Rereading configuration. ]
[ Mar 26 19:47:56 Rereading configuration. ]
[ Mar 26 19:48:00 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:48:00.656297774Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:48:00.659260283Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:48:00.659277634Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:48:00.859678409Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:49:12.349652172Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:57999"}
{"msg":"region 70630c85-1f34-41f1-9e6d-85a187376eec state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:49:12.39617191Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:49:12.39915337Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"uri":"/crucible/0/regions","method":"POST","req_id":"78ce9f83-9484-4771-b7b5-f5a4466f4bc0","remote_addr":"[fd00:1122:3344:101::a]:57999","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":18155,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:49:12.401508591Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/70630c85-1f34-41f1-9e6d-85a187376eec","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:49:12.411085671Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/70630c85-1f34-41f1-9e6d-85a187376eec","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:49:12.411116822Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"70630c85-1f34-41f1-9e6d-85a187376eec\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/70630c85-1f34-41f1-9e6d-85a187376eec\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:49:12.457239688Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"region":"70630c85-1f34-41f1-9e6d-85a187376eec","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:49:12.698111588Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"uri":"/crucible/0/regions","method":"POST","req_id":"109fc339-8b3b-46b2-a7e7-35a3e9934b6f","remote_addr":"[fd00:1122:3344:101::a]:57999","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":77,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:49:12.744091489Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"region":"70630c85-1f34-41f1-9e6d-85a187376eec","component":"worker"}
{"msg":"region 70630c85-1f34-41f1-9e6d-85a187376eec state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:49:12.74411536Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:49:12.744237274Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-70630c85-1f34-41f1-9e6d-85a187376eec","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:49:12.744640866Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-70630c85-1f34-41f1-9e6d-85a187376eec","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:49:12.749476583Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:49:12.763489707Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-70630c85-1f34-41f1-9e6d-85a187376eec","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:49:12.766467577Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/70630c85-1f34-41f1-9e6d-85a187376eec","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:49:12.767039554Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:49:12.769310053Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:49:12.769414876Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:49:12.769428696Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:49:12.778353226Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:49:12.784480562Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:49:13.006407388Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"uri":"/crucible/0/regions","method":"POST","req_id":"4a612f69-0ef8-44fa-a609-ec9ca545090c","remote_addr":"[fd00:1122:3344:101::a]:57999","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":79,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:42.752144962Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:53283"}
{"msg":"region eee3be5c-efbd-4392-b273-66d6ae4fe3c8 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:42.813421477Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:42.81418839Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"uri":"/crucible/0/regions","method":"POST","req_id":"cee483b3-92a0-42fe-9c90-edb7f06145b2","remote_addr":"[fd00:1122:3344:101::a]:53283","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":819,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:42.814209Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/eee3be5c-efbd-4392-b273-66d6ae4fe3c8","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:42.824199953Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/eee3be5c-efbd-4392-b273-66d6ae4fe3c8","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:42.824235734Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"eee3be5c-efbd-4392-b273-66d6ae4fe3c8\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/eee3be5c-efbd-4392-b273-66d6ae4fe3c8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:42.871985349Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"region":"eee3be5c-efbd-4392-b273-66d6ae4fe3c8","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:42.94008852Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"region":"eee3be5c-efbd-4392-b273-66d6ae4fe3c8","component":"worker"}
{"msg":"region eee3be5c-efbd-4392-b273-66d6ae4fe3c8 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:42.940132041Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:42.940233794Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/70630c85-1f34-41f1-9e6d-85a187376eec","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:42.952501666Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:42.952589588Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:42.952673931Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:42.952682441Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:42.952724522Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-70630c85-1f34-41f1-9e6d-85a187376eec","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:42.952777524Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/70630c85-1f34-41f1-9e6d-85a187376eec","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:42.952875647Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:42.952942419Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:42.95299119Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:42.953040352Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:42.96090368Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-eee3be5c-efbd-4392-b273-66d6ae4fe3c8","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:42.961425645Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-eee3be5c-efbd-4392-b273-66d6ae4fe3c8","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:42.96453494Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:42.973428259Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-eee3be5c-efbd-4392-b273-66d6ae4fe3c8","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:42.976248374Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/eee3be5c-efbd-4392-b273-66d6ae4fe3c8","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:42.976581844Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:42.976639606Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:42.976680807Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:42.976723179Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:42.980295167Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:42.985628058Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:43.038486268Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"uri":"/crucible/0/regions","method":"POST","req_id":"07954c21-71ff-455e-8be9-eb1d7cf4d963","remote_addr":"[fd00:1122:3344:101::a]:53283","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":81,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:44.061028091Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:51754"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:44.061117184Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"uri":"/crucible/0/regions/70630c85-1f34-41f1-9e6d-85a187376eec","method":"GET","req_id":"d6c8c4e3-0b74-40bd-8e2a-ff7b582bc39b","remote_addr":"[fd00:1122:3344:101::a]:51754","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":75,"response_code":"200"}
{"msg":"path is \"/data/regions/70630c85-1f34-41f1-9e6d-85a187376eec\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:44.061262988Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"datafile"}
{"msg":"dataset is oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/70630c85-1f34-41f1-9e6d-85a187376eec","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:44.071165588Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"datafile"}
{"msg":"snapshot is oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/70630c85-1f34-41f1-9e6d-85a187376eec@b602aa42-71bd-47aa-96a7-d6be5868de11","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:44.080208052Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/70630c85-1f34-41f1-9e6d-85a187376eec\", \"b602aa42-71bd-47aa-96a7-d6be5868de11\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:44.080235622Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:44.091559845Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"uri":"/crucible/0/regions/70630c85-1f34-41f1-9e6d-85a187376eec/snapshots/b602aa42-71bd-47aa-96a7-d6be5868de11","method":"GET","req_id":"3e3440fe-f0a0-43fe-9415-705263de8470","remote_addr":"[fd00:1122:3344:101::a]:51754","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":30304,"response_code":"200"}
{"msg":"path is \"/data/regions/70630c85-1f34-41f1-9e6d-85a187376eec\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:44.09171218Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"datafile"}
{"msg":"dataset is oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/70630c85-1f34-41f1-9e6d-85a187376eec","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:44.100103334Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"datafile"}
{"msg":"snapshot is oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/70630c85-1f34-41f1-9e6d-85a187376eec@b602aa42-71bd-47aa-96a7-d6be5868de11","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:44.107654772Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/70630c85-1f34-41f1-9e6d-85a187376eec\", \"b602aa42-71bd-47aa-96a7-d6be5868de11\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:44.107693823Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 70630c85-1f34-41f1-9e6d-85a187376eec-b602aa42-71bd-47aa-96a7-d6be5868de11 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:44.117385837Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"datafile"}
{"msg":"applying SMF actions for region 70630c85-1f34-41f1-9e6d-85a187376eec running snapshot b602aa42-71bd-47aa-96a7-d6be5868de11 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:44.117557492Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:44.117588703Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"uri":"/crucible/0/regions/70630c85-1f34-41f1-9e6d-85a187376eec/snapshots/b602aa42-71bd-47aa-96a7-d6be5868de11/run","method":"POST","req_id":"346d788c-307f-4b91-995c-97246ecace93","remote_addr":"[fd00:1122:3344:101::a]:51754","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":25912,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/70630c85-1f34-41f1-9e6d-85a187376eec","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:44.118269323Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:44.118376347Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:44.11847818Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:44.1184857Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:44.118537992Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-70630c85-1f34-41f1-9e6d-85a187376eec","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:44.118585833Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/70630c85-1f34-41f1-9e6d-85a187376eec","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:44.118952744Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:44.119017226Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:44.119067388Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:44.119122329Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:44.122483981Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-70630c85-1f34-41f1-9e6d-85a187376eec-b602aa42-71bd-47aa-96a7-d6be5868de11","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:44.123902174Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-70630c85-1f34-41f1-9e6d-85a187376eec-b602aa42-71bd-47aa-96a7-d6be5868de11","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:44.128021719Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-70630c85-1f34-41f1-9e6d-85a187376eec-b602aa42-71bd-47aa-96a7-d6be5868de11","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:44.138297258Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/70630c85-1f34-41f1-9e6d-85a187376eec/.zfs/snapshot/b602aa42-71bd-47aa-96a7-d6be5868de11","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:44.138611898Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:44.1386837Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:44.138723842Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:44.138762613Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:44.138813674Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:44.142183136Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:44.146377583Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"worker"}
{"msg":"region 70630c85-1f34-41f1-9e6d-85a187376eec running snapshot b602aa42-71bd-47aa-96a7-d6be5868de11 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-03-26T19:54:44.146397074Z","hostname":"oxz_crucible_2cac9179-a451-44f0-8e8a-fcbb6bc2fb93","pid":23175,"component":"datafile"}