[ Mar 21 18:03:38 Enabled. ]
[ Mar 21 18:03:38 Rereading configuration. ]
[ Mar 21 18:03:39 Rereading configuration. ]
[ Mar 21 18:03:43 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crucible -l [fd00:1122:3344:101::17]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:03:43.157971546Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000}
{"msg":"listen IP: [fd00:1122:3344:101::17]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:03:43.161662142Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:03:43.161685613Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:03:43.288041997Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.301760959Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:55179"}
{"msg":"region 7bd351a8-f34b-44ab-9289-818aa53413e5 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.31411692Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.315075877Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"uri":"/crucible/0/regions","method":"POST","req_id":"32f084b5-8637-4e36-af02-9adb6f11e4b3","remote_addr":"[fd00:1122:3344:101::a]:55179","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":6221,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.315125168Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crucible/regions/7bd351a8-f34b-44ab-9289-818aa53413e5","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.325478834Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crucible/regions/7bd351a8-f34b-44ab-9289-818aa53413e5","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.325544485Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"7bd351a8-f34b-44ab-9289-818aa53413e5\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/7bd351a8-f34b-44ab-9289-818aa53413e5\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.394196815Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"region":"7bd351a8-f34b-44ab-9289-818aa53413e5","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.496375566Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"uri":"/crucible/0/regions","method":"POST","req_id":"fe962ef9-c061-4c32-ad70-a88d297d92e3","remote_addr":"[fd00:1122:3344:101::a]:55179","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":608,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.544482638Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"region":"7bd351a8-f34b-44ab-9289-818aa53413e5","component":"worker"}
{"msg":"region 7bd351a8-f34b-44ab-9289-818aa53413e5 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.544510019Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.544785514Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-7bd351a8-f34b-44ab-9289-818aa53413e5","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.545255992Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-7bd351a8-f34b-44ab-9289-818aa53413e5","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.549302825Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.56074303Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-7bd351a8-f34b-44ab-9289-818aa53413e5","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.564146851Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/7bd351a8-f34b-44ab-9289-818aa53413e5","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.564907944Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.564985406Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.565030817Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.565066627Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.569362024Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.578756443Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.826435762Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"uri":"/crucible/0/regions","method":"POST","req_id":"c118b863-5a0b-4066-9871-d2f61f28e687","remote_addr":"[fd00:1122:3344:101::a]:55179","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":113,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:02.898811986Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:33247"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:02.898925978Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"uri":"/crucible/0/regions/7bd351a8-f34b-44ab-9289-818aa53413e5","method":"GET","req_id":"0b32e8ec-5611-4e72-a956-2b35e824180b","remote_addr":"[fd00:1122:3344:101::a]:33247","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":87,"response_code":"200"}
{"msg":"path is \"/data/regions/7bd351a8-f34b-44ab-9289-818aa53413e5\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:02.899163662Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"component":"datafile"}
{"msg":"dataset is oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crucible/regions/7bd351a8-f34b-44ab-9289-818aa53413e5","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:02.908469419Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"component":"datafile"}
{"msg":"snapshot is oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crucible/regions/7bd351a8-f34b-44ab-9289-818aa53413e5@fe1e7c32-a3e0-469a-9e5c-10034bec5870","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:02.917428419Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crucible/regions/7bd351a8-f34b-44ab-9289-818aa53413e5\", \"fe1e7c32-a3e0-469a-9e5c-10034bec5870\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:02.91745713Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:02.931746366Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"uri":"/crucible/0/regions/7bd351a8-f34b-44ab-9289-818aa53413e5/snapshots/fe1e7c32-a3e0-469a-9e5c-10034bec5870","method":"GET","req_id":"922b0830-dab5-48f0-b073-81359802a60e","remote_addr":"[fd00:1122:3344:101::a]:33247","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":32609,"response_code":"200"}
{"msg":"path is \"/data/regions/7bd351a8-f34b-44ab-9289-818aa53413e5\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:02.931904159Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"component":"datafile"}
{"msg":"dataset is oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crucible/regions/7bd351a8-f34b-44ab-9289-818aa53413e5","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:02.940846879Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"component":"datafile"}
{"msg":"snapshot is oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crucible/regions/7bd351a8-f34b-44ab-9289-818aa53413e5@fe1e7c32-a3e0-469a-9e5c-10034bec5870","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:02.949173599Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crucible/regions/7bd351a8-f34b-44ab-9289-818aa53413e5\", \"fe1e7c32-a3e0-469a-9e5c-10034bec5870\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:02.949205269Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 7bd351a8-f34b-44ab-9289-818aa53413e5-fe1e7c32-a3e0-469a-9e5c-10034bec5870 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:02.958276642Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"component":"datafile"}
{"msg":"applying SMF actions for region 7bd351a8-f34b-44ab-9289-818aa53413e5 running snapshot fe1e7c32-a3e0-469a-9e5c-10034bec5870 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:02.958833142Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:02.958882853Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"uri":"/crucible/0/regions/7bd351a8-f34b-44ab-9289-818aa53413e5/snapshots/fe1e7c32-a3e0-469a-9e5c-10034bec5870/run","method":"POST","req_id":"257eb0de-1b2a-42cb-a3eb-f9544ff2a048","remote_addr":"[fd00:1122:3344:101::a]:33247","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":27004,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-7bd351a8-f34b-44ab-9289-818aa53413e5-fe1e7c32-a3e0-469a-9e5c-10034bec5870","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:02.970409799Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-7bd351a8-f34b-44ab-9289-818aa53413e5-fe1e7c32-a3e0-469a-9e5c-10034bec5870","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:02.974335559Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-7bd351a8-f34b-44ab-9289-818aa53413e5-fe1e7c32-a3e0-469a-9e5c-10034bec5870","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:02.985973088Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/7bd351a8-f34b-44ab-9289-818aa53413e5/.zfs/snapshot/fe1e7c32-a3e0-469a-9e5c-10034bec5870","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:02.986245663Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:02.986308764Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:02.986351035Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:02.986392466Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:02.986453247Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:02.990191324Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:02.994633813Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"component":"worker"}
{"msg":"region 7bd351a8-f34b-44ab-9289-818aa53413e5 running snapshot fe1e7c32-a3e0-469a-9e5c-10034bec5870 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:02.994645323Z","hostname":"oxz_crucible_65a251b5-db44-4383-ada6-83f1fde9bd43","pid":23000,"component":"datafile"}