[ Jul 10 20:02:49 Enabled. ]
[ Jul 10 20:02:49 Rereading configuration. ]
[ Jul 10 20:02:50 Rereading configuration. ]
[ Jul 10 20:02:51 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_9c15209c-5a20-4ab1-875a-73276f0ce3ed/crucible -l [fd00:1122:3344:101::12]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_9c15209c-5a20-4ab1-875a-73276f0ce3ed/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:02:51.914463741Z","hostname":"oxz_crucible_291b6da3-6167-4fe4-96ed-cf3af4696367","pid":26903}
{"msg":"listen IP: [fd00:1122:3344:101::12]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:02:51.917775739Z","hostname":"oxz_crucible_291b6da3-6167-4fe4-96ed-cf3af4696367","pid":26903}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:02:51.917789779Z","hostname":"oxz_crucible_291b6da3-6167-4fe4-96ed-cf3af4696367","pid":26903}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:02:51.932646882Z","hostname":"oxz_crucible_291b6da3-6167-4fe4-96ed-cf3af4696367","pid":26903,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:02:52.034131743Z","hostname":"oxz_crucible_291b6da3-6167-4fe4-96ed-cf3af4696367","pid":26903,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.536696304Z","hostname":"oxz_crucible_291b6da3-6167-4fe4-96ed-cf3af4696367","pid":26903,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:42992"}
{"msg":"region 0e490934-0d54-45d1-806a-af32e3e128bb state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.54586696Z","hostname":"oxz_crucible_291b6da3-6167-4fe4-96ed-cf3af4696367","pid":26903,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.550020894Z","hostname":"oxz_crucible_291b6da3-6167-4fe4-96ed-cf3af4696367","pid":26903,"uri":"/crucible/0/regions","method":"POST","req_id":"2d4b133c-620d-4fd8-b690-2927a83a628f","remote_addr":"[fd00:1122:3344:101::c]:42992","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":8287,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.550662839Z","hostname":"oxz_crucible_291b6da3-6167-4fe4-96ed-cf3af4696367","pid":26903,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_9c15209c-5a20-4ab1-875a-73276f0ce3ed/crucible/regions/0e490934-0d54-45d1-806a-af32e3e128bb","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.560841694Z","hostname":"oxz_crucible_291b6da3-6167-4fe4-96ed-cf3af4696367","pid":26903,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_9c15209c-5a20-4ab1-875a-73276f0ce3ed/crucible/regions/0e490934-0d54-45d1-806a-af32e3e128bb","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.560876674Z","hostname":"oxz_crucible_291b6da3-6167-4fe4-96ed-cf3af4696367","pid":26903,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"0e490934-0d54-45d1-806a-af32e3e128bb\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/0e490934-0d54-45d1-806a-af32e3e128bb\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.689744032Z","hostname":"oxz_crucible_291b6da3-6167-4fe4-96ed-cf3af4696367","pid":26903,"region":"0e490934-0d54-45d1-806a-af32e3e128bb","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"0e490934-0d54-45d1-806a-af32e3e128bb\" \"--data\" \"/data/regions/0e490934-0d54-45d1-806a-af32e3e128bb\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.691789379Z","hostname":"oxz_crucible_291b6da3-6167-4fe4-96ed-cf3af4696367","pid":26903,"region":"0e490934-0d54-45d1-806a-af32e3e128bb","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.786916248Z","hostname":"oxz_crucible_291b6da3-6167-4fe4-96ed-cf3af4696367","pid":26903,"uri":"/crucible/0/regions","method":"POST","req_id":"4f49f19d-931c-4e7e-860b-ab5a75e7a494","remote_addr":"[fd00:1122:3344:101::c]:42992","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":88,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.910285779Z","hostname":"oxz_crucible_291b6da3-6167-4fe4-96ed-cf3af4696367","pid":26903,"region":"0e490934-0d54-45d1-806a-af32e3e128bb","component":"worker"}
{"msg":"region 0e490934-0d54-45d1-806a-af32e3e128bb state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.91032623Z","hostname":"oxz_crucible_291b6da3-6167-4fe4-96ed-cf3af4696367","pid":26903,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.910435371Z","hostname":"oxz_crucible_291b6da3-6167-4fe4-96ed-cf3af4696367","pid":26903,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-0e490934-0d54-45d1-806a-af32e3e128bb","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.910852744Z","hostname":"oxz_crucible_291b6da3-6167-4fe4-96ed-cf3af4696367","pid":26903,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-0e490934-0d54-45d1-806a-af32e3e128bb","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.91878983Z","hostname":"oxz_crucible_291b6da3-6167-4fe4-96ed-cf3af4696367","pid":26903,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.959411366Z","hostname":"oxz_crucible_291b6da3-6167-4fe4-96ed-cf3af4696367","pid":26903,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-0e490934-0d54-45d1-806a-af32e3e128bb","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.980947175Z","hostname":"oxz_crucible_291b6da3-6167-4fe4-96ed-cf3af4696367","pid":26903,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/0e490934-0d54-45d1-806a-af32e3e128bb","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.98156966Z","hostname":"oxz_crucible_291b6da3-6167-4fe4-96ed-cf3af4696367","pid":26903,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.985441172Z","hostname":"oxz_crucible_291b6da3-6167-4fe4-96ed-cf3af4696367","pid":26903,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.985491462Z","hostname":"oxz_crucible_291b6da3-6167-4fe4-96ed-cf3af4696367","pid":26903,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:13.985533593Z","hostname":"oxz_crucible_291b6da3-6167-4fe4-96ed-cf3af4696367","pid":26903,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:14.018609457Z","hostname":"oxz_crucible_291b6da3-6167-4fe4-96ed-cf3af4696367","pid":26903,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:14.030503446Z","hostname":"oxz_crucible_291b6da3-6167-4fe4-96ed-cf3af4696367","pid":26903,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:14.138651292Z","hostname":"oxz_crucible_291b6da3-6167-4fe4-96ed-cf3af4696367","pid":26903,"uri":"/crucible/0/regions","method":"POST","req_id":"0c811d24-57a8-4add-9617-e341ace2e491","remote_addr":"[fd00:1122:3344:101::c]:42992","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:14.976391094Z","hostname":"oxz_crucible_291b6da3-6167-4fe4-96ed-cf3af4696367","pid":26903,"uri":"/crucible/0/regions/0e490934-0d54-45d1-806a-af32e3e128bb","method":"GET","req_id":"bbb2c03c-300f-4417-8aaf-b4f581d4f17f","remote_addr":"[fd00:1122:3344:101::c]:42992","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":127,"response_code":"200"}
{"msg":"path is \"/data/regions/0e490934-0d54-45d1-806a-af32e3e128bb\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:14.978607752Z","hostname":"oxz_crucible_291b6da3-6167-4fe4-96ed-cf3af4696367","pid":26903,"component":"datafile"}
{"msg":"dataset is oxp_9c15209c-5a20-4ab1-875a-73276f0ce3ed/crucible/regions/0e490934-0d54-45d1-806a-af32e3e128bb","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:14.987674578Z","hostname":"oxz_crucible_291b6da3-6167-4fe4-96ed-cf3af4696367","pid":26903,"component":"datafile"}
{"msg":"region 0e490934-0d54-45d1-806a-af32e3e128bb state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:14.996460471Z","hostname":"oxz_crucible_291b6da3-6167-4fe4-96ed-cf3af4696367","pid":26903,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:14.996603001Z","hostname":"oxz_crucible_291b6da3-6167-4fe4-96ed-cf3af4696367","pid":26903,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:14.996663402Z","hostname":"oxz_crucible_291b6da3-6167-4fe4-96ed-cf3af4696367","pid":26903,"uri":"/crucible/0/regions/0e490934-0d54-45d1-806a-af32e3e128bb","method":"DELETE","req_id":"671f9e71-e339-4e19-bf35-d9ae4f6a7f3a","remote_addr":"[fd00:1122:3344:101::c]:42992","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":18117,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-0e490934-0d54-45d1-806a-af32e3e128bb (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:14.99764697Z","hostname":"oxz_crucible_291b6da3-6167-4fe4-96ed-cf3af4696367","pid":26903,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:14.998748509Z","hostname":"oxz_crucible_291b6da3-6167-4fe4-96ed-cf3af4696367","pid":26903,"uri":"/crucible/0/regions/0e490934-0d54-45d1-806a-af32e3e128bb","method":"GET","req_id":"a014bfbb-2d2d-40da-bb74-6ad19355b0d2","remote_addr":"[fd00:1122:3344:101::c]:42992","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":28,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:15.007721394Z","hostname":"oxz_crucible_291b6da3-6167-4fe4-96ed-cf3af4696367","pid":26903,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_9c15209c-5a20-4ab1-875a-73276f0ce3ed/crucible/regions/0e490934-0d54-45d1-806a-af32e3e128bb\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:15.017567785Z","hostname":"oxz_crucible_291b6da3-6167-4fe4-96ed-cf3af4696367","pid":26903,"region":"0e490934-0d54-45d1-806a-af32e3e128bb","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:15.113605001Z","hostname":"oxz_crucible_291b6da3-6167-4fe4-96ed-cf3af4696367","pid":26903,"uri":"/crucible/0/regions/0e490934-0d54-45d1-806a-af32e3e128bb","method":"GET","req_id":"ecf63828-4806-426b-8ab6-65cde043e6bb","remote_addr":"[fd00:1122:3344:101::c]:42992","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":92,"response_code":"200"}
{"msg":"region 0e490934-0d54-45d1-806a-af32e3e128bb state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:15.128334953Z","hostname":"oxz_crucible_291b6da3-6167-4fe4-96ed-cf3af4696367","pid":26903,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-10T20:03:15.253228438Z","hostname":"oxz_crucible_291b6da3-6167-4fe4-96ed-cf3af4696367","pid":26903,"uri":"/crucible/0/regions/0e490934-0d54-45d1-806a-af32e3e128bb","method":"GET","req_id":"23be15f8-5cb1-4f9e-a3a2-8768e4dbc9c5","remote_addr":"[fd00:1122:3344:101::c]:42992","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":457,"response_code":"200"}