[ Jan 22 09:42:07 Enabled. ]
[ Jan 22 09:42:07 Rereading configuration. ]
[ Jan 22 09:42:12 Rereading configuration. ]
[ Jan 22 09:42:23 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_bc6fb756-0db4-4cd5-8758-408750a53dc8/crucible -l [fd00:1122:3344:101::1a]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_bc6fb756-0db4-4cd5-8758-408750a53dc8/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:42:23.843399206Z","hostname":"oxz_crucible_030024ef-f258-4cd4-8ca6-c5ff1e1ffdb8","pid":22836}
{"msg":"listen IP: [fd00:1122:3344:101::1a]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:42:23.853890831Z","hostname":"oxz_crucible_030024ef-f258-4cd4-8ca6-c5ff1e1ffdb8","pid":22836}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:42:23.853922342Z","hostname":"oxz_crucible_030024ef-f258-4cd4-8ca6-c5ff1e1ffdb8","pid":22836}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:42:23.853942582Z","hostname":"oxz_crucible_030024ef-f258-4cd4-8ca6-c5ff1e1ffdb8","pid":22836,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:42:24.102334011Z","hostname":"oxz_crucible_030024ef-f258-4cd4-8ca6-c5ff1e1ffdb8","pid":22836,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:43:02.740202126Z","hostname":"oxz_crucible_030024ef-f258-4cd4-8ca6-c5ff1e1ffdb8","pid":22836,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:42409"}
{"msg":"region 03977efb-93a8-4dd1-89db-c42d3b9ac935 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:43:02.782809498Z","hostname":"oxz_crucible_030024ef-f258-4cd4-8ca6-c5ff1e1ffdb8","pid":22836,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:43:02.788338961Z","hostname":"oxz_crucible_030024ef-f258-4cd4-8ca6-c5ff1e1ffdb8","pid":22836,"uri":"/crucible/0/regions","method":"POST","req_id":"fe7b2600-4074-4810-a69c-6013b3e06b83","remote_addr":"[fd00:1122:3344:101::c]:42409","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":17756,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:43:02.791204634Z","hostname":"oxz_crucible_030024ef-f258-4cd4-8ca6-c5ff1e1ffdb8","pid":22836,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_bc6fb756-0db4-4cd5-8758-408750a53dc8/crucible/regions/03977efb-93a8-4dd1-89db-c42d3b9ac935","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:43:02.801201391Z","hostname":"oxz_crucible_030024ef-f258-4cd4-8ca6-c5ff1e1ffdb8","pid":22836,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_bc6fb756-0db4-4cd5-8758-408750a53dc8/crucible/regions/03977efb-93a8-4dd1-89db-c42d3b9ac935","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:43:02.801264122Z","hostname":"oxz_crucible_030024ef-f258-4cd4-8ca6-c5ff1e1ffdb8","pid":22836,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"03977efb-93a8-4dd1-89db-c42d3b9ac935\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/03977efb-93a8-4dd1-89db-c42d3b9ac935\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:43:02.866973713Z","hostname":"oxz_crucible_030024ef-f258-4cd4-8ca6-c5ff1e1ffdb8","pid":22836,"region":"03977efb-93a8-4dd1-89db-c42d3b9ac935","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"03977efb-93a8-4dd1-89db-c42d3b9ac935\" \"--data\" \"/data/regions/03977efb-93a8-4dd1-89db-c42d3b9ac935\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:43:02.869426998Z","hostname":"oxz_crucible_030024ef-f258-4cd4-8ca6-c5ff1e1ffdb8","pid":22836,"region":"03977efb-93a8-4dd1-89db-c42d3b9ac935","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:43:03.040159463Z","hostname":"oxz_crucible_030024ef-f258-4cd4-8ca6-c5ff1e1ffdb8","pid":22836,"uri":"/crucible/0/regions","method":"POST","req_id":"43a467d1-60ad-4e81-b5c2-bcd7ad912c8f","remote_addr":"[fd00:1122:3344:101::c]:42409","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":92,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:43:03.111309186Z","hostname":"oxz_crucible_030024ef-f258-4cd4-8ca6-c5ff1e1ffdb8","pid":22836,"region":"03977efb-93a8-4dd1-89db-c42d3b9ac935","component":"worker"}
{"msg":"region 03977efb-93a8-4dd1-89db-c42d3b9ac935 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:43:03.111360477Z","hostname":"oxz_crucible_030024ef-f258-4cd4-8ca6-c5ff1e1ffdb8","pid":22836,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:43:03.111447598Z","hostname":"oxz_crucible_030024ef-f258-4cd4-8ca6-c5ff1e1ffdb8","pid":22836,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-03977efb-93a8-4dd1-89db-c42d3b9ac935","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:43:03.111832566Z","hostname":"oxz_crucible_030024ef-f258-4cd4-8ca6-c5ff1e1ffdb8","pid":22836,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-03977efb-93a8-4dd1-89db-c42d3b9ac935","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:43:03.118810685Z","hostname":"oxz_crucible_030024ef-f258-4cd4-8ca6-c5ff1e1ffdb8","pid":22836,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:43:03.133243233Z","hostname":"oxz_crucible_030024ef-f258-4cd4-8ca6-c5ff1e1ffdb8","pid":22836,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-03977efb-93a8-4dd1-89db-c42d3b9ac935","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:43:03.148392935Z","hostname":"oxz_crucible_030024ef-f258-4cd4-8ca6-c5ff1e1ffdb8","pid":22836,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/03977efb-93a8-4dd1-89db-c42d3b9ac935","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:43:03.148920395Z","hostname":"oxz_crucible_030024ef-f258-4cd4-8ca6-c5ff1e1ffdb8","pid":22836,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:43:03.150832621Z","hostname":"oxz_crucible_030024ef-f258-4cd4-8ca6-c5ff1e1ffdb8","pid":22836,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:43:03.150886052Z","hostname":"oxz_crucible_030024ef-f258-4cd4-8ca6-c5ff1e1ffdb8","pid":22836,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:43:03.150928872Z","hostname":"oxz_crucible_030024ef-f258-4cd4-8ca6-c5ff1e1ffdb8","pid":22836,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:43:03.155191392Z","hostname":"oxz_crucible_030024ef-f258-4cd4-8ca6-c5ff1e1ffdb8","pid":22836,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:43:03.159522872Z","hostname":"oxz_crucible_030024ef-f258-4cd4-8ca6-c5ff1e1ffdb8","pid":22836,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:43:03.584617956Z","hostname":"oxz_crucible_030024ef-f258-4cd4-8ca6-c5ff1e1ffdb8","pid":22836,"uri":"/crucible/0/regions","method":"POST","req_id":"0aff359b-afc7-426b-9ad3-2c6afd69e15e","remote_addr":"[fd00:1122:3344:101::c]:42409","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":76,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:43:03.95380657Z","hostname":"oxz_crucible_030024ef-f258-4cd4-8ca6-c5ff1e1ffdb8","pid":22836,"uri":"/crucible/0/regions/03977efb-93a8-4dd1-89db-c42d3b9ac935","method":"GET","req_id":"a1a26a92-5112-4f1a-ab00-464e2fb531ce","remote_addr":"[fd00:1122:3344:101::c]:42409","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"path is \"/data/regions/03977efb-93a8-4dd1-89db-c42d3b9ac935\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:43:03.956357877Z","hostname":"oxz_crucible_030024ef-f258-4cd4-8ca6-c5ff1e1ffdb8","pid":22836,"component":"datafile"}
{"msg":"dataset is oxp_bc6fb756-0db4-4cd5-8758-408750a53dc8/crucible/regions/03977efb-93a8-4dd1-89db-c42d3b9ac935","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:43:03.967550225Z","hostname":"oxz_crucible_030024ef-f258-4cd4-8ca6-c5ff1e1ffdb8","pid":22836,"component":"datafile"}
{"msg":"region 03977efb-93a8-4dd1-89db-c42d3b9ac935 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:43:03.977370798Z","hostname":"oxz_crucible_030024ef-f258-4cd4-8ca6-c5ff1e1ffdb8","pid":22836,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:43:03.977538981Z","hostname":"oxz_crucible_030024ef-f258-4cd4-8ca6-c5ff1e1ffdb8","pid":22836,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-03977efb-93a8-4dd1-89db-c42d3b9ac935 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:43:03.978759344Z","hostname":"oxz_crucible_030024ef-f258-4cd4-8ca6-c5ff1e1ffdb8","pid":22836,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:43:03.979825654Z","hostname":"oxz_crucible_030024ef-f258-4cd4-8ca6-c5ff1e1ffdb8","pid":22836,"uri":"/crucible/0/regions/03977efb-93a8-4dd1-89db-c42d3b9ac935","method":"DELETE","req_id":"c1c61fcd-a960-4cc1-95ef-eddb9460c9a7","remote_addr":"[fd00:1122:3344:101::c]:42409","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":23515,"response_code":"204"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:43:03.983253777Z","hostname":"oxz_crucible_030024ef-f258-4cd4-8ca6-c5ff1e1ffdb8","pid":22836,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:43:03.986427256Z","hostname":"oxz_crucible_030024ef-f258-4cd4-8ca6-c5ff1e1ffdb8","pid":22836,"uri":"/crucible/0/regions/03977efb-93a8-4dd1-89db-c42d3b9ac935","method":"GET","req_id":"e7575c2f-f7e0-426b-a228-edc127c8bcdc","remote_addr":"[fd00:1122:3344:101::c]:42409","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":639,"response_code":"200"}
{"msg":"deleting zfs dataset \"oxp_bc6fb756-0db4-4cd5-8758-408750a53dc8/crucible/regions/03977efb-93a8-4dd1-89db-c42d3b9ac935\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:43:03.995387723Z","hostname":"oxz_crucible_030024ef-f258-4cd4-8ca6-c5ff1e1ffdb8","pid":22836,"region":"03977efb-93a8-4dd1-89db-c42d3b9ac935","component":"worker"}
{"msg":"region 03977efb-93a8-4dd1-89db-c42d3b9ac935 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:43:04.064384346Z","hostname":"oxz_crucible_030024ef-f258-4cd4-8ca6-c5ff1e1ffdb8","pid":22836,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:43:04.121267974Z","hostname":"oxz_crucible_030024ef-f258-4cd4-8ca6-c5ff1e1ffdb8","pid":22836,"uri":"/crucible/0/regions/03977efb-93a8-4dd1-89db-c42d3b9ac935","method":"GET","req_id":"365bcb65-9353-4154-8866-f51a50271b25","remote_addr":"[fd00:1122:3344:101::c]:42409","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":87,"response_code":"200"}