[ Oct 30 19:45:48 Enabled. ]
[ Oct 30 19:45:48 Rereading configuration. ]
[ Oct 30 19:45:52 Rereading configuration. ]
[ Oct 30 19:45:54 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_0c2810be-f07e-4592-922b-2bf8e520a121/crucible -l [fd00:1122:3344:101::1a]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_0c2810be-f07e-4592-922b-2bf8e520a121/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:45:54.861902197Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684}
{"msg":"listen IP: [fd00:1122:3344:101::1a]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:45:54.8650135Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:45:54.86502299Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:45:54.879486426Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:45:54.998633241Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:50.736458595Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:56075"}
{"msg":"region dbbf88ac-2282-4b0c-9230-7604e674773b state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:50.775210678Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:50.77981924Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"uri":"/crucible/0/regions","method":"POST","req_id":"94779d96-90db-45b4-bc90-bcb430ec6400","remote_addr":"[fd00:1122:3344:101::a]:56075","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":13857,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:50.784270479Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_0c2810be-f07e-4592-922b-2bf8e520a121/crucible/regions/dbbf88ac-2282-4b0c-9230-7604e674773b","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:50.794165383Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_0c2810be-f07e-4592-922b-2bf8e520a121/crucible/regions/dbbf88ac-2282-4b0c-9230-7604e674773b","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:50.794201634Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"dbbf88ac-2282-4b0c-9230-7604e674773b\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/dbbf88ac-2282-4b0c-9230-7604e674773b\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:50.842662255Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"region":"dbbf88ac-2282-4b0c-9230-7604e674773b","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"dbbf88ac-2282-4b0c-9230-7604e674773b\" \"--data\" \"/data/regions/dbbf88ac-2282-4b0c-9230-7604e674773b\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:50.848177382Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"region":"dbbf88ac-2282-4b0c-9230-7604e674773b","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:51.031631761Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"uri":"/crucible/0/regions","method":"POST","req_id":"67a59ed0-378a-4a58-b617-c61d8dfe8007","remote_addr":"[fd00:1122:3344:101::a]:56075","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":76,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:51.14341855Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"region":"dbbf88ac-2282-4b0c-9230-7604e674773b","component":"worker"}
{"msg":"region dbbf88ac-2282-4b0c-9230-7604e674773b state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:51.143471842Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:51.143576824Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-dbbf88ac-2282-4b0c-9230-7604e674773b","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:51.146214325Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-dbbf88ac-2282-4b0c-9230-7604e674773b","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:51.149930033Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:51.168361954Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-dbbf88ac-2282-4b0c-9230-7604e674773b","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:51.17121236Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/dbbf88ac-2282-4b0c-9230-7604e674773b","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:51.179312346Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:51.179371578Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:51.179412189Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:51.17945578Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:51.183252471Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:51.187934006Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:51.62337653Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"uri":"/crucible/0/regions","method":"POST","req_id":"c5f01b00-d239-4f85-b82f-c40b214ec032","remote_addr":"[fd00:1122:3344:101::a]:56075","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":65,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:52.029797251Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"uri":"/crucible/0/regions/dbbf88ac-2282-4b0c-9230-7604e674773b","method":"GET","req_id":"8e1a5975-f8f3-4090-a6d6-4edebdbdb7eb","remote_addr":"[fd00:1122:3344:101::a]:56075","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":92,"response_code":"200"}
{"msg":"path is \"/data/regions/dbbf88ac-2282-4b0c-9230-7604e674773b\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:52.032743319Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"datafile"}
{"msg":"dataset is oxp_0c2810be-f07e-4592-922b-2bf8e520a121/crucible/regions/dbbf88ac-2282-4b0c-9230-7604e674773b","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:52.044327308Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"datafile"}
{"msg":"region dbbf88ac-2282-4b0c-9230-7604e674773b state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:52.054630803Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:52.054789827Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:52.054878829Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"uri":"/crucible/0/regions/dbbf88ac-2282-4b0c-9230-7604e674773b","method":"DELETE","req_id":"64af9173-b3c6-4192-b121-608a3c7f558e","remote_addr":"[fd00:1122:3344:101::a]:56075","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":22162,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-dbbf88ac-2282-4b0c-9230-7604e674773b (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:52.055989619Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:52.061328751Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_0c2810be-f07e-4592-922b-2bf8e520a121/crucible/regions/dbbf88ac-2282-4b0c-9230-7604e674773b\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:52.073029543Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"region":"dbbf88ac-2282-4b0c-9230-7604e674773b","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:52.120238961Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"uri":"/crucible/0/regions/dbbf88ac-2282-4b0c-9230-7604e674773b","method":"GET","req_id":"6200ff73-d5ef-427b-91f1-bcf21e9bf943","remote_addr":"[fd00:1122:3344:101::a]:56075","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":82,"response_code":"200"}
{"msg":"region dbbf88ac-2282-4b0c-9230-7604e674773b state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:52.133933816Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:52.209621143Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"uri":"/crucible/0/regions/dbbf88ac-2282-4b0c-9230-7604e674773b","method":"GET","req_id":"9466cd7a-f9a3-4bed-9e0f-69ba945197f2","remote_addr":"[fd00:1122:3344:101::a]:56075","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":36,"response_code":"200"}
{"msg":"region db7bc268-4281-4776-9a4d-27e08e3c79bc state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:54.314184425Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:54.31434055Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"uri":"/crucible/0/regions","method":"POST","req_id":"99cdee29-319d-475f-9406-27a006303d05","remote_addr":"[fd00:1122:3344:101::a]:56075","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":224,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:54.31436366Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_0c2810be-f07e-4592-922b-2bf8e520a121/crucible/regions/db7bc268-4281-4776-9a4d-27e08e3c79bc","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:54.324237113Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_0c2810be-f07e-4592-922b-2bf8e520a121/crucible/regions/db7bc268-4281-4776-9a4d-27e08e3c79bc","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:54.324272054Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"db7bc268-4281-4776-9a4d-27e08e3c79bc\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/db7bc268-4281-4776-9a4d-27e08e3c79bc\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:54.376559418Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"region":"db7bc268-4281-4776-9a4d-27e08e3c79bc","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"db7bc268-4281-4776-9a4d-27e08e3c79bc\" \"--data\" \"/data/regions/db7bc268-4281-4776-9a4d-27e08e3c79bc\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:54.376612639Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"region":"db7bc268-4281-4776-9a4d-27e08e3c79bc","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:54.454306239Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"region":"db7bc268-4281-4776-9a4d-27e08e3c79bc","component":"worker"}
{"msg":"region db7bc268-4281-4776-9a4d-27e08e3c79bc state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:54.454360741Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:54.454441893Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-db7bc268-4281-4776-9a4d-27e08e3c79bc","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:54.455862051Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-db7bc268-4281-4776-9a4d-27e08e3c79bc","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:54.462469867Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:54.473923882Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-db7bc268-4281-4776-9a4d-27e08e3c79bc","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:54.477861497Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/db7bc268-4281-4776-9a4d-27e08e3c79bc","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:54.478174186Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:54.478232317Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:54.478283669Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:54.47832588Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:54.482135671Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:54.486747564Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:54.54776766Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"uri":"/crucible/0/regions","method":"POST","req_id":"e1634fbe-219c-4f4d-a5a1-eab5bfe68fae","remote_addr":"[fd00:1122:3344:101::a]:56075","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:17.254151822Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:64295"}
{"msg":"region c82cce0c-b433-45ef-8087-9344c4263882 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:17.254343568Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:17.254850451Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"uri":"/crucible/0/regions","method":"POST","req_id":"8fa9a072-3b91-4ed4-a4ad-e799d287385a","remote_addr":"[fd00:1122:3344:101::a]:64295","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":652,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:17.254869542Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_0c2810be-f07e-4592-922b-2bf8e520a121/crucible/regions/c82cce0c-b433-45ef-8087-9344c4263882","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:17.265400932Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_0c2810be-f07e-4592-922b-2bf8e520a121/crucible/regions/c82cce0c-b433-45ef-8087-9344c4263882","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:17.265435163Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"c82cce0c-b433-45ef-8087-9344c4263882\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/c82cce0c-b433-45ef-8087-9344c4263882\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:17.324198399Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"region":"c82cce0c-b433-45ef-8087-9344c4263882","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"c82cce0c-b433-45ef-8087-9344c4263882\" \"--data\" \"/data/regions/c82cce0c-b433-45ef-8087-9344c4263882\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:17.324269961Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"region":"c82cce0c-b433-45ef-8087-9344c4263882","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:17.405299301Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"region":"c82cce0c-b433-45ef-8087-9344c4263882","component":"worker"}
{"msg":"region c82cce0c-b433-45ef-8087-9344c4263882 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:17.405346432Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:17.405431034Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-c82cce0c-b433-45ef-8087-9344c4263882","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:17.40640564Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-c82cce0c-b433-45ef-8087-9344c4263882","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:17.415289957Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:17.427986445Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-c82cce0c-b433-45ef-8087-9344c4263882","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:17.431499419Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c82cce0c-b433-45ef-8087-9344c4263882","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:17.431769926Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:17.431834788Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:17.431878639Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:17.43191996Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:17.43606774Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:17.441986958Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:17.463903972Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"uri":"/crucible/0/regions","method":"POST","req_id":"2896e036-7cd4-4cea-86bc-20558902f12b","remote_addr":"[fd00:1122:3344:101::a]:64295","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:19.599035811Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"uri":"/crucible/0/regions/db7bc268-4281-4776-9a4d-27e08e3c79bc","method":"GET","req_id":"e73f7192-85aa-467d-8193-d5acfadb6f6a","remote_addr":"[fd00:1122:3344:101::a]:64295","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":54,"response_code":"200"}
{"msg":"path is \"/data/regions/db7bc268-4281-4776-9a4d-27e08e3c79bc\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:19.602671938Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"datafile"}
{"msg":"dataset is oxp_0c2810be-f07e-4592-922b-2bf8e520a121/crucible/regions/db7bc268-4281-4776-9a4d-27e08e3c79bc","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:19.612201752Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"datafile"}
{"msg":"snapshot is oxp_0c2810be-f07e-4592-922b-2bf8e520a121/crucible/regions/db7bc268-4281-4776-9a4d-27e08e3c79bc@be7d4cc9-ad0b-41d7-9186-e57217c23c9f","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:19.620969905Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_0c2810be-f07e-4592-922b-2bf8e520a121/crucible/regions/db7bc268-4281-4776-9a4d-27e08e3c79bc\", \"be7d4cc9-ad0b-41d7-9186-e57217c23c9f\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:19.621000756Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:19.634433824Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"uri":"/crucible/0/regions/db7bc268-4281-4776-9a4d-27e08e3c79bc/snapshots/be7d4cc9-ad0b-41d7-9186-e57217c23c9f","method":"GET","req_id":"9ab87768-586b-43ac-852c-773eb1ea8ffe","remote_addr":"[fd00:1122:3344:101::a]:64295","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":31779,"response_code":"200"}
{"msg":"path is \"/data/regions/db7bc268-4281-4776-9a4d-27e08e3c79bc\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:19.638249136Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"datafile"}
{"msg":"dataset is oxp_0c2810be-f07e-4592-922b-2bf8e520a121/crucible/regions/db7bc268-4281-4776-9a4d-27e08e3c79bc","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:19.646770203Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"datafile"}
{"msg":"snapshot is oxp_0c2810be-f07e-4592-922b-2bf8e520a121/crucible/regions/db7bc268-4281-4776-9a4d-27e08e3c79bc@be7d4cc9-ad0b-41d7-9186-e57217c23c9f","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:19.654205261Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_0c2810be-f07e-4592-922b-2bf8e520a121/crucible/regions/db7bc268-4281-4776-9a4d-27e08e3c79bc\", \"be7d4cc9-ad0b-41d7-9186-e57217c23c9f\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:19.654238532Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot db7bc268-4281-4776-9a4d-27e08e3c79bc-be7d4cc9-ad0b-41d7-9186-e57217c23c9f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:19.663299623Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"datafile"}
{"msg":"applying SMF actions for region db7bc268-4281-4776-9a4d-27e08e3c79bc running snapshot be7d4cc9-ad0b-41d7-9186-e57217c23c9f (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:19.663489028Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:19.663514569Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"uri":"/crucible/0/regions/db7bc268-4281-4776-9a4d-27e08e3c79bc/snapshots/be7d4cc9-ad0b-41d7-9186-e57217c23c9f/run","method":"POST","req_id":"81da2c88-959e-4dfe-8783-09834330819f","remote_addr":"[fd00:1122:3344:101::a]:64295","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":25302,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-db7bc268-4281-4776-9a4d-27e08e3c79bc-be7d4cc9-ad0b-41d7-9186-e57217c23c9f","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:19.666282103Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-db7bc268-4281-4776-9a4d-27e08e3c79bc-be7d4cc9-ad0b-41d7-9186-e57217c23c9f","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:19.669083037Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-db7bc268-4281-4776-9a4d-27e08e3c79bc-be7d4cc9-ad0b-41d7-9186-e57217c23c9f","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:19.679396532Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/db7bc268-4281-4776-9a4d-27e08e3c79bc/.zfs/snapshot/be7d4cc9-ad0b-41d7-9186-e57217c23c9f","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:19.679633029Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:19.67968247Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:19.679727591Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:19.679770272Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:19.679822014Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:19.683489341Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:19.688105674Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"worker"}
{"msg":"region db7bc268-4281-4776-9a4d-27e08e3c79bc running snapshot be7d4cc9-ad0b-41d7-9186-e57217c23c9f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:19.688121425Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"datafile"}
{"msg":"path is \"/data/regions/db7bc268-4281-4776-9a4d-27e08e3c79bc\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:19.936651298Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"datafile"}
{"msg":"dataset is oxp_0c2810be-f07e-4592-922b-2bf8e520a121/crucible/regions/db7bc268-4281-4776-9a4d-27e08e3c79bc","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:19.94575622Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"datafile"}
{"msg":"snapshot is oxp_0c2810be-f07e-4592-922b-2bf8e520a121/crucible/regions/db7bc268-4281-4776-9a4d-27e08e3c79bc@be7d4cc9-ad0b-41d7-9186-e57217c23c9f","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:19.954554375Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_0c2810be-f07e-4592-922b-2bf8e520a121/crucible/regions/db7bc268-4281-4776-9a4d-27e08e3c79bc\", \"be7d4cc9-ad0b-41d7-9186-e57217c23c9f\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:19.954590596Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:50:19.962042464Z","hostname":"oxz_crucible_fc41377f-86d4-4e2b-a9f4-12b2a390cebf","pid":26684,"uri":"/crucible/0/regions/db7bc268-4281-4776-9a4d-27e08e3c79bc/snapshots/be7d4cc9-ad0b-41d7-9186-e57217c23c9f/run","method":"POST","req_id":"a90134a2-797f-4ab3-9fe3-4fcf810f4ca5","remote_addr":"[fd00:1122:3344:101::a]:64295","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":25994,"response_code":"200"}