[ Apr 26 00:20:11 Enabled. ]
[ Apr 26 00:20:11 Rereading configuration. ]
[ Apr 26 00:20:13 Rereading configuration. ]
[ Apr 26 00:20:15 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:20:15.456272172Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:20:15.45864863Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:20:15.458664731Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:20:15.481602399Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:20:15.563362753Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:21:26.567506057Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:49607"}
{"msg":"region cd3940ae-6535-43d6-b671-d13b52e33b2c state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:21:26.579220555Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:21:26.582225274Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:21:26.586403761Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"uri":"/crucible/0/regions","method":"POST","req_id":"0d0e0653-114c-409e-99d7-3ff0f2403838","remote_addr":"[fd00:1122:3344:101::a]:49607","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":14643,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f/crucible/regions/cd3940ae-6535-43d6-b671-d13b52e33b2c","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:21:26.593085678Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f/crucible/regions/cd3940ae-6535-43d6-b671-d13b52e33b2c","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:21:26.593121058Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"cd3940ae-6535-43d6-b671-d13b52e33b2c\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/cd3940ae-6535-43d6-b671-d13b52e33b2c\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:21:26.643445337Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"region":"cd3940ae-6535-43d6-b671-d13b52e33b2c","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"cd3940ae-6535-43d6-b671-d13b52e33b2c\" \"--data\" \"/data/regions/cd3940ae-6535-43d6-b671-d13b52e33b2c\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:21:26.645386908Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"region":"cd3940ae-6535-43d6-b671-d13b52e33b2c","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:21:26.778799301Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"uri":"/crucible/0/regions","method":"POST","req_id":"50947195-4310-4dd2-a4a1-3bb97aba0f49","remote_addr":"[fd00:1122:3344:101::a]:49607","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":88,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:21:26.788855673Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"region":"cd3940ae-6535-43d6-b671-d13b52e33b2c","component":"worker"}
{"msg":"region cd3940ae-6535-43d6-b671-d13b52e33b2c state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:21:26.788884923Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:21:26.789000465Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-cd3940ae-6535-43d6-b671-d13b52e33b2c","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:21:26.791458565Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-cd3940ae-6535-43d6-b671-d13b52e33b2c","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:21:26.795706963Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:21:26.80673492Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-cd3940ae-6535-43d6-b671-d13b52e33b2c","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:21:26.809636937Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/cd3940ae-6535-43d6-b671-d13b52e33b2c","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:21:26.810343198Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:21:26.812430261Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:21:26.812488962Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:21:26.812532093Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:21:26.820546452Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:21:26.824830241Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:21:27.221440233Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"uri":"/crucible/0/regions","method":"POST","req_id":"b5db14f8-c7ca-4a1a-9ecb-97e3c31a0e9d","remote_addr":"[fd00:1122:3344:101::a]:49607","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":104,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:56.844952229Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:59906"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:56.848601448Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"uri":"/crucible/0/regions/cd3940ae-6535-43d6-b671-d13b52e33b2c","method":"GET","req_id":"31aea310-57e3-40cb-bd97-a7cafce7abe5","remote_addr":"[fd00:1122:3344:101::a]:59906","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":3606,"response_code":"200"}
{"msg":"path is \"/data/regions/cd3940ae-6535-43d6-b671-d13b52e33b2c\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:56.851362062Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"datafile"}
{"msg":"dataset is oxp_1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f/crucible/regions/cd3940ae-6535-43d6-b671-d13b52e33b2c","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:56.85995269Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"datafile"}
{"msg":"snapshot is oxp_1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f/crucible/regions/cd3940ae-6535-43d6-b671-d13b52e33b2c@8a8cb26d-a12c-47cb-a73a-5f6e4573a290","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:56.867634693Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f/crucible/regions/cd3940ae-6535-43d6-b671-d13b52e33b2c\", \"8a8cb26d-a12c-47cb-a73a-5f6e4573a290\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:56.867662554Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:56.878894694Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"uri":"/crucible/0/regions/cd3940ae-6535-43d6-b671-d13b52e33b2c/snapshots/8a8cb26d-a12c-47cb-a73a-5f6e4573a290","method":"GET","req_id":"d0ff351e-e7bf-43b0-bb5d-7fe3f7840f7d","remote_addr":"[fd00:1122:3344:101::a]:59906","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":30107,"response_code":"200"}
{"msg":"path is \"/data/regions/cd3940ae-6535-43d6-b671-d13b52e33b2c\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:56.879089968Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"datafile"}
{"msg":"dataset is oxp_1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f/crucible/regions/cd3940ae-6535-43d6-b671-d13b52e33b2c","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:56.887229938Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"datafile"}
{"msg":"snapshot is oxp_1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f/crucible/regions/cd3940ae-6535-43d6-b671-d13b52e33b2c@8a8cb26d-a12c-47cb-a73a-5f6e4573a290","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:56.895048244Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f/crucible/regions/cd3940ae-6535-43d6-b671-d13b52e33b2c\", \"8a8cb26d-a12c-47cb-a73a-5f6e4573a290\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:56.895083154Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot cd3940ae-6535-43d6-b671-d13b52e33b2c-8a8cb26d-a12c-47cb-a73a-5f6e4573a290 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:56.904317663Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"datafile"}
{"msg":"applying SMF actions for region cd3940ae-6535-43d6-b671-d13b52e33b2c running snapshot 8a8cb26d-a12c-47cb-a73a-5f6e4573a290 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:56.904937773Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:56.907140278Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"uri":"/crucible/0/regions/cd3940ae-6535-43d6-b671-d13b52e33b2c/snapshots/8a8cb26d-a12c-47cb-a73a-5f6e4573a290/run","method":"POST","req_id":"6480d6cf-8cbe-44d3-901a-247653db9369","remote_addr":"[fd00:1122:3344:101::a]:59906","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":28074,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/cd3940ae-6535-43d6-b671-d13b52e33b2c","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:56.914263553Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:56.914351884Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:56.914442216Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:56.914449516Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:56.914494906Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-cd3940ae-6535-43d6-b671-d13b52e33b2c","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:56.914543157Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/cd3940ae-6535-43d6-b671-d13b52e33b2c","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:56.914636919Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:56.91470635Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:56.914746941Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:56.914814662Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:56.922108899Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-cd3940ae-6535-43d6-b671-d13b52e33b2c-8a8cb26d-a12c-47cb-a73a-5f6e4573a290","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:56.922604167Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-cd3940ae-6535-43d6-b671-d13b52e33b2c-8a8cb26d-a12c-47cb-a73a-5f6e4573a290","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:56.926737003Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-cd3940ae-6535-43d6-b671-d13b52e33b2c-8a8cb26d-a12c-47cb-a73a-5f6e4573a290","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:56.940740948Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/cd3940ae-6535-43d6-b671-d13b52e33b2c/.zfs/snapshot/8a8cb26d-a12c-47cb-a73a-5f6e4573a290","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:56.941063173Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:56.941124074Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:56.941166115Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:56.941206355Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:56.941258446Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:56.9458652Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:56.949961846Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"region cd3940ae-6535-43d6-b671-d13b52e33b2c running snapshot 8a8cb26d-a12c-47cb-a73a-5f6e4573a290 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:56.949989057Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"datafile"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:59.873013418Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:49500"}
{"msg":"region e0d3da98-9672-4c42-81cd-966008aebdb6 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:59.903311695Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:59.903470537Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"uri":"/crucible/0/regions","method":"POST","req_id":"b77adfa8-8168-43fe-bf9f-6188ba068ead","remote_addr":"[fd00:1122:3344:101::a]:49500","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":691,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:59.903497078Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f/crucible/regions/e0d3da98-9672-4c42-81cd-966008aebdb6","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:59.913438857Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f/crucible/regions/e0d3da98-9672-4c42-81cd-966008aebdb6","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:59.913462518Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"e0d3da98-9672-4c42-81cd-966008aebdb6\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/e0d3da98-9672-4c42-81cd-966008aebdb6\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:59.984649641Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"region":"e0d3da98-9672-4c42-81cd-966008aebdb6","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"e0d3da98-9672-4c42-81cd-966008aebdb6\" \"--data\" \"/data/regions/e0d3da98-9672-4c42-81cd-966008aebdb6\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:26:59.986403539Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"region":"e0d3da98-9672-4c42-81cd-966008aebdb6","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:00.056060669Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"region":"e0d3da98-9672-4c42-81cd-966008aebdb6","component":"worker"}
{"msg":"region e0d3da98-9672-4c42-81cd-966008aebdb6 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:00.05610829Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:00.056242542Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/cd3940ae-6535-43d6-b671-d13b52e33b2c","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:00.056828311Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:00.056921252Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:00.057004074Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:00.057012904Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:00.057056515Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-cd3940ae-6535-43d6-b671-d13b52e33b2c","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:00.057125456Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/cd3940ae-6535-43d6-b671-d13b52e33b2c","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:00.057544293Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:00.057610094Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:00.057663275Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:00.057712445Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:00.061188881Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-e0d3da98-9672-4c42-81cd-966008aebdb6","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:00.061594058Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-e0d3da98-9672-4c42-81cd-966008aebdb6","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:00.067710216Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:00.075779755Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e0d3da98-9672-4c42-81cd-966008aebdb6","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:00.080137585Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e0d3da98-9672-4c42-81cd-966008aebdb6","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:00.08039448Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:00.080469531Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:00.080514171Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:00.080555952Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:00.08538542Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:00.09165633Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:00.2758513Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"uri":"/crucible/0/regions","method":"POST","req_id":"5ba03bf6-6ccc-47ab-803c-d3a582ffb1f0","remote_addr":"[fd00:1122:3344:101::a]:49500","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":68,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:29.589436931Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:43898"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:29.589497882Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"uri":"/crucible/0/regions/e0d3da98-9672-4c42-81cd-966008aebdb6","method":"GET","req_id":"0ff275c7-c77d-4b7c-83a7-7064168f8aef","remote_addr":"[fd00:1122:3344:101::a]:43898","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":45,"response_code":"200"}
{"msg":"path is \"/data/regions/e0d3da98-9672-4c42-81cd-966008aebdb6\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:29.589650014Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"datafile"}
{"msg":"dataset is oxp_1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f/crucible/regions/e0d3da98-9672-4c42-81cd-966008aebdb6","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:29.601296801Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"datafile"}
{"msg":"region e0d3da98-9672-4c42-81cd-966008aebdb6 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:29.610981167Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:29.61120207Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-e0d3da98-9672-4c42-81cd-966008aebdb6 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:29.612637044Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:29.613427166Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"uri":"/crucible/0/regions/e0d3da98-9672-4c42-81cd-966008aebdb6","method":"DELETE","req_id":"50f23b31-3103-4cd9-94fe-1e65645cea44","remote_addr":"[fd00:1122:3344:101::a]:43898","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":23805,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:29.61431519Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"uri":"/crucible/0/regions/e0d3da98-9672-4c42-81cd-966008aebdb6","method":"GET","req_id":"495a6a70-cdde-4266-92b1-600e2c8a868a","remote_addr":"[fd00:1122:3344:101::a]:43898","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":745,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/cd3940ae-6535-43d6-b671-d13b52e33b2c","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:29.621967663Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:29.622062225Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:29.622158586Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:29.622171857Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:29.622225098Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-cd3940ae-6535-43d6-b671-d13b52e33b2c","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:29.622279558Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/cd3940ae-6535-43d6-b671-d13b52e33b2c","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:29.622628084Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:29.622693915Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:29.622744816Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:29.622794587Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:29.631693539Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:29.634003227Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_1af63a9d-d9ad-43e0-9ba8-d7354e1b6a8f/crucible/regions/e0d3da98-9672-4c42-81cd-966008aebdb6\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:29.644096349Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"region":"e0d3da98-9672-4c42-81cd-966008aebdb6","component":"worker"}
{"msg":"region e0d3da98-9672-4c42-81cd-966008aebdb6 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:29.695627966Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-26T00:27:29.728688468Z","hostname":"oxz_crucible_5145b222-0ffb-403b-a919-074d655b57e3","pid":23747,"uri":"/crucible/0/regions/e0d3da98-9672-4c42-81cd-966008aebdb6","method":"GET","req_id":"ca5e52d0-d450-4767-801a-d7284c4edbff","remote_addr":"[fd00:1122:3344:101::a]:43898","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":108,"response_code":"200"}