[ Jul 13 05:35:51 Enabled. ]
[ Jul 13 05:35:51 Rereading configuration. ]
[ Jul 13 05:35:53 Rereading configuration. ]
[ Jul 13 05:35:56 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_59a1b65b-0e8e-4af6-9fcd-02021a3973de/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_59a1b65b-0e8e-4af6-9fcd-02021a3973de/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:35:56.223528014Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:35:56.227786891Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:35:56.227804611Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:35:56.247760082Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:35:56.58347394Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:37:10.762445388Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:63889"}
{"msg":"region d70635da-c833-4c73-87b2-27c0905f6896 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:37:10.786396571Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:37:10.794351195Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"uri":"/crucible/0/regions","method":"POST","req_id":"6ac075b1-7252-4df9-a161-b7b9cb16d4a8","remote_addr":"[fd00:1122:3344:101::a]:63889","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":17598,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:37:10.796851404Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_59a1b65b-0e8e-4af6-9fcd-02021a3973de/crucible/regions/d70635da-c833-4c73-87b2-27c0905f6896","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:37:10.809227497Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_59a1b65b-0e8e-4af6-9fcd-02021a3973de/crucible/regions/d70635da-c833-4c73-87b2-27c0905f6896","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:37:10.809263617Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"d70635da-c833-4c73-87b2-27c0905f6896\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/d70635da-c833-4c73-87b2-27c0905f6896\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:37:10.874666436Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"region":"d70635da-c833-4c73-87b2-27c0905f6896","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"d70635da-c833-4c73-87b2-27c0905f6896\" \"--data\" \"/data/regions/d70635da-c833-4c73-87b2-27c0905f6896\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:37:10.877160815Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"region":"d70635da-c833-4c73-87b2-27c0905f6896","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:37:11.055768806Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"region":"d70635da-c833-4c73-87b2-27c0905f6896","component":"worker"}
{"msg":"region d70635da-c833-4c73-87b2-27c0905f6896 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:37:11.055827007Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:37:11.055898738Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-d70635da-c833-4c73-87b2-27c0905f6896","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:37:11.056303294Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-d70635da-c833-4c73-87b2-27c0905f6896","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:37:11.063019859Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:37:11.079553432Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-d70635da-c833-4c73-87b2-27c0905f6896","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:37:11.082400381Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/d70635da-c833-4c73-87b2-27c0905f6896","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:37:11.083237474Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:37:11.088033509Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:37:11.088097799Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"uri":"/crucible/0/regions","method":"POST","req_id":"06a792eb-a9be-46f7-b699-207a6f1d9eac","remote_addr":"[fd00:1122:3344:101::a]:63889","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":2333,"response_code":"200"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:37:11.08811707Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:37:11.08815199Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:37:11.094879055Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:37:11.101467848Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:29.040369519Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:37684"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:29.043789282Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"uri":"/crucible/0/regions/d70635da-c833-4c73-87b2-27c0905f6896","method":"GET","req_id":"1f3322d8-a660-4bf4-b3da-8bb1fe40f492","remote_addr":"[fd00:1122:3344:101::a]:37684","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":3397,"response_code":"200"}
{"msg":"path is \"/data/regions/d70635da-c833-4c73-87b2-27c0905f6896\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:29.046390562Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"datafile"}
{"msg":"dataset is oxp_59a1b65b-0e8e-4af6-9fcd-02021a3973de/crucible/regions/d70635da-c833-4c73-87b2-27c0905f6896","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:29.055600936Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"datafile"}
{"msg":"snapshot is oxp_59a1b65b-0e8e-4af6-9fcd-02021a3973de/crucible/regions/d70635da-c833-4c73-87b2-27c0905f6896@5d2d3acf-ae8e-42b2-be21-a0630b25a89a","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:29.065032352Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_59a1b65b-0e8e-4af6-9fcd-02021a3973de/crucible/regions/d70635da-c833-4c73-87b2-27c0905f6896\", \"5d2d3acf-ae8e-42b2-be21-a0630b25a89a\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:29.065059493Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:29.077189562Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"uri":"/crucible/0/regions/d70635da-c833-4c73-87b2-27c0905f6896/snapshots/5d2d3acf-ae8e-42b2-be21-a0630b25a89a","method":"GET","req_id":"8452d6b1-8f20-48a3-b2f5-6595a5feac86","remote_addr":"[fd00:1122:3344:101::a]:37684","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":33217,"response_code":"200"}
{"msg":"path is \"/data/regions/d70635da-c833-4c73-87b2-27c0905f6896\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:29.077333334Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"datafile"}
{"msg":"dataset is oxp_59a1b65b-0e8e-4af6-9fcd-02021a3973de/crucible/regions/d70635da-c833-4c73-87b2-27c0905f6896","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:29.08604024Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"datafile"}
{"msg":"snapshot is oxp_59a1b65b-0e8e-4af6-9fcd-02021a3973de/crucible/regions/d70635da-c833-4c73-87b2-27c0905f6896@5d2d3acf-ae8e-42b2-be21-a0630b25a89a","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:29.094875367Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_59a1b65b-0e8e-4af6-9fcd-02021a3973de/crucible/regions/d70635da-c833-4c73-87b2-27c0905f6896\", \"5d2d3acf-ae8e-42b2-be21-a0630b25a89a\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:29.094910108Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot d70635da-c833-4c73-87b2-27c0905f6896-5d2d3acf-ae8e-42b2-be21-a0630b25a89a state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:29.104225103Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"datafile"}
{"msg":"applying SMF actions for region d70635da-c833-4c73-87b2-27c0905f6896 running snapshot 5d2d3acf-ae8e-42b2-be21-a0630b25a89a (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:29.105073796Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:29.105140127Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"uri":"/crucible/0/regions/d70635da-c833-4c73-87b2-27c0905f6896/snapshots/5d2d3acf-ae8e-42b2-be21-a0630b25a89a/run","method":"POST","req_id":"23ad6054-da02-41f5-9817-9dad20828802","remote_addr":"[fd00:1122:3344:101::a]:37684","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":27828,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/d70635da-c833-4c73-87b2-27c0905f6896","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:29.117110854Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:29.117216155Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:29.117315917Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:29.117323787Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:29.117376418Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-d70635da-c833-4c73-87b2-27c0905f6896","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:29.117432828Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/d70635da-c833-4c73-87b2-27c0905f6896","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:29.11754153Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:29.117602141Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:29.117651542Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:29.117700963Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:29.125797089Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-d70635da-c833-4c73-87b2-27c0905f6896-5d2d3acf-ae8e-42b2-be21-a0630b25a89a","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:29.126374618Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-d70635da-c833-4c73-87b2-27c0905f6896-5d2d3acf-ae8e-42b2-be21-a0630b25a89a","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:29.131866323Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-d70635da-c833-4c73-87b2-27c0905f6896-5d2d3acf-ae8e-42b2-be21-a0630b25a89a","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:29.152729378Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/d70635da-c833-4c73-87b2-27c0905f6896/.zfs/snapshot/5d2d3acf-ae8e-42b2-be21-a0630b25a89a","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:29.153065734Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:29.153128635Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:29.153169295Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:29.153210136Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:29.153269487Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:29.160455469Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:29.168562964Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"region d70635da-c833-4c73-87b2-27c0905f6896 running snapshot 5d2d3acf-ae8e-42b2-be21-a0630b25a89a state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:29.168579954Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"datafile"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:32.12676901Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:63439"}
{"msg":"region 73a48ee4-d45a-4bef-bdd1-5c3ffc7fc931 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:32.126869462Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:32.129376291Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:32.129385281Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"uri":"/crucible/0/regions","method":"POST","req_id":"a48f8ed0-128f-4dab-88d8-18f2c3fe6f2c","remote_addr":"[fd00:1122:3344:101::a]:63439","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":2553,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_59a1b65b-0e8e-4af6-9fcd-02021a3973de/crucible/regions/73a48ee4-d45a-4bef-bdd1-5c3ffc7fc931","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:32.138997841Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_59a1b65b-0e8e-4af6-9fcd-02021a3973de/crucible/regions/73a48ee4-d45a-4bef-bdd1-5c3ffc7fc931","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:32.139033141Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"73a48ee4-d45a-4bef-bdd1-5c3ffc7fc931\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/73a48ee4-d45a-4bef-bdd1-5c3ffc7fc931\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:32.206924709Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"region":"73a48ee4-d45a-4bef-bdd1-5c3ffc7fc931","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"73a48ee4-d45a-4bef-bdd1-5c3ffc7fc931\" \"--data\" \"/data/regions/73a48ee4-d45a-4bef-bdd1-5c3ffc7fc931\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:32.20697939Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"region":"73a48ee4-d45a-4bef-bdd1-5c3ffc7fc931","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:32.332033767Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"region":"73a48ee4-d45a-4bef-bdd1-5c3ffc7fc931","component":"worker"}
{"msg":"region 73a48ee4-d45a-4bef-bdd1-5c3ffc7fc931 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:32.332086828Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:32.332167149Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-73a48ee4-d45a-4bef-bdd1-5c3ffc7fc931","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:32.332601626Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-73a48ee4-d45a-4bef-bdd1-5c3ffc7fc931","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:32.342583561Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:32.347395806Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"uri":"/crucible/0/regions","method":"POST","req_id":"bcb33452-591d-4497-89c4-433dc6c8675c","remote_addr":"[fd00:1122:3344:101::a]:63439","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":78,"response_code":"200"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:32.359298822Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-73a48ee4-d45a-4bef-bdd1-5c3ffc7fc931","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:32.36630064Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/73a48ee4-d45a-4bef-bdd1-5c3ffc7fc931","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:32.366517574Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:32.366565535Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:32.366608155Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:32.366647226Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:32.37461174Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/d70635da-c833-4c73-87b2-27c0905f6896","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:32.381087891Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:32.381193963Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:32.381295824Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:32.381314434Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:32.381354555Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-d70635da-c833-4c73-87b2-27c0905f6896","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:32.381410366Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/d70635da-c833-4c73-87b2-27c0905f6896","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:32.387635873Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:32.387688044Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:32.387736594Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:32.387784015Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:32.398506212Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:32.400419042Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:58.514031526Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"uri":"/crucible/0/regions/73a48ee4-d45a-4bef-bdd1-5c3ffc7fc931","method":"GET","req_id":"cee27305-23e2-477f-9585-128de0bc9d91","remote_addr":"[fd00:1122:3344:101::a]:63439","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":61,"response_code":"200"}
{"msg":"path is \"/data/regions/73a48ee4-d45a-4bef-bdd1-5c3ffc7fc931\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:58.516498034Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"datafile"}
{"msg":"dataset is oxp_59a1b65b-0e8e-4af6-9fcd-02021a3973de/crucible/regions/73a48ee4-d45a-4bef-bdd1-5c3ffc7fc931","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:58.527703869Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"datafile"}
{"msg":"region 73a48ee4-d45a-4bef-bdd1-5c3ffc7fc931 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:58.536260752Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:58.536480576Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:58.536565217Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"uri":"/crucible/0/regions/73a48ee4-d45a-4bef-bdd1-5c3ffc7fc931","method":"DELETE","req_id":"9f55570a-d8a2-4249-9d3c-8b3872307634","remote_addr":"[fd00:1122:3344:101::a]:63439","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":20094,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-73a48ee4-d45a-4bef-bdd1-5c3ffc7fc931 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:58.537777796Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:58.539561764Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"uri":"/crucible/0/regions/73a48ee4-d45a-4bef-bdd1-5c3ffc7fc931","method":"GET","req_id":"90b289bb-91ce-4c6b-8754-d6329d030c55","remote_addr":"[fd00:1122:3344:101::a]:63439","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":26,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/d70635da-c833-4c73-87b2-27c0905f6896","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:58.542103253Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:58.542189775Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:58.542275346Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:58.542285916Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:58.542331757Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-d70635da-c833-4c73-87b2-27c0905f6896","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:58.542387718Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/d70635da-c833-4c73-87b2-27c0905f6896","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:58.542752623Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:58.542829445Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:58.542887436Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:58.542935516Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:58.5476817Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:58.549121993Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_59a1b65b-0e8e-4af6-9fcd-02021a3973de/crucible/regions/73a48ee4-d45a-4bef-bdd1-5c3ffc7fc931\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:58.559067388Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"region":"73a48ee4-d45a-4bef-bdd1-5c3ffc7fc931","component":"worker"}
{"msg":"region 73a48ee4-d45a-4bef-bdd1-5c3ffc7fc931 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:58.605814216Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-13T05:42:58.690146499Z","hostname":"oxz_crucible_b54516c2-071a-40de-86b7-fe7645854290","pid":22033,"uri":"/crucible/0/regions/73a48ee4-d45a-4bef-bdd1-5c3ffc7fc931","method":"GET","req_id":"bffd117a-3d94-4456-9b9f-905aa0785ca7","remote_addr":"[fd00:1122:3344:101::a]:63439","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":59,"response_code":"200"}