[ Nov 25 19:31:02 Enabled. ]
[ Nov 25 19:31:02 Rereading configuration. ]
[ Nov 25 19:31:05 Rereading configuration. ]
[ Nov 25 19:31:09 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_1b4c5120-bb11-444d-9c86-edb04be1264c/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_1b4c5120-bb11-444d-9c86-edb04be1264c/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:09.90050446Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:09.904033059Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:09.904043629Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:09.921054683Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:09.99381878Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.581769916Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:48914"}
{"msg":"region fe2e91d3-eafe-44ee-bcd3-ec2a10dd793f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.612636631Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.616615429Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.621289781Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"uri":"/crucible/0/regions","method":"POST","req_id":"3dbb36ed-fe8c-4ad5-9735-85cee120fbde","remote_addr":"[fd00:1122:3344:101::a]:48914","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":14355,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_1b4c5120-bb11-444d-9c86-edb04be1264c/crucible/regions/fe2e91d3-eafe-44ee-bcd3-ec2a10dd793f","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.627928431Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_1b4c5120-bb11-444d-9c86-edb04be1264c/crucible/regions/fe2e91d3-eafe-44ee-bcd3-ec2a10dd793f","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.627952132Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"fe2e91d3-eafe-44ee-bcd3-ec2a10dd793f\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/fe2e91d3-eafe-44ee-bcd3-ec2a10dd793f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.681053454Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"region":"fe2e91d3-eafe-44ee-bcd3-ec2a10dd793f","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"fe2e91d3-eafe-44ee-bcd3-ec2a10dd793f\" \"--data\" \"/data/regions/fe2e91d3-eafe-44ee-bcd3-ec2a10dd793f\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.682958001Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"region":"fe2e91d3-eafe-44ee-bcd3-ec2a10dd793f","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.807066296Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"region":"fe2e91d3-eafe-44ee-bcd3-ec2a10dd793f","component":"worker"}
{"msg":"region fe2e91d3-eafe-44ee-bcd3-ec2a10dd793f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.807114027Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.807199849Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-fe2e91d3-eafe-44ee-bcd3-ec2a10dd793f","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.807596986Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-fe2e91d3-eafe-44ee-bcd3-ec2a10dd793f","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.811962882Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.822398187Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-fe2e91d3-eafe-44ee-bcd3-ec2a10dd793f","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.825496888Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/fe2e91d3-eafe-44ee-bcd3-ec2a10dd793f","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.826207471Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.826280593Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.826335214Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.826377105Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.829907114Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.833890112Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:32:19.834491204Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"uri":"/crucible/0/regions","method":"POST","req_id":"60e9ff9f-c95b-4faf-b1d5-de6a2b0a7f63","remote_addr":"[fd00:1122:3344:101::a]:48914","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":70,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.306317436Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:61003"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.309425957Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"uri":"/crucible/0/regions/fe2e91d3-eafe-44ee-bcd3-ec2a10dd793f","method":"GET","req_id":"ccbc74d7-c2a7-4547-88b8-864e11ff8d42","remote_addr":"[fd00:1122:3344:101::a]:61003","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":3065,"response_code":"200"}
{"msg":"path is \"/data/regions/fe2e91d3-eafe-44ee-bcd3-ec2a10dd793f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.312948036Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"datafile"}
{"msg":"dataset is oxp_1b4c5120-bb11-444d-9c86-edb04be1264c/crucible/regions/fe2e91d3-eafe-44ee-bcd3-ec2a10dd793f","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.321727638Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"datafile"}
{"msg":"snapshot is oxp_1b4c5120-bb11-444d-9c86-edb04be1264c/crucible/regions/fe2e91d3-eafe-44ee-bcd3-ec2a10dd793f@a71d0d30-1bc4-4c84-887d-7dc2bb733294","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.330056652Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_1b4c5120-bb11-444d-9c86-edb04be1264c/crucible/regions/fe2e91d3-eafe-44ee-bcd3-ec2a10dd793f\", \"a71d0d30-1bc4-4c84-887d-7dc2bb733294\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.330089563Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.340415985Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"uri":"/crucible/0/regions/fe2e91d3-eafe-44ee-bcd3-ec2a10dd793f/snapshots/a71d0d30-1bc4-4c84-887d-7dc2bb733294","method":"GET","req_id":"322e7b36-d0c8-4dcb-9a64-aa48d01680b9","remote_addr":"[fd00:1122:3344:101::a]:61003","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":27475,"response_code":"200"}
{"msg":"path is \"/data/regions/fe2e91d3-eafe-44ee-bcd3-ec2a10dd793f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.344199939Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"datafile"}
{"msg":"dataset is oxp_1b4c5120-bb11-444d-9c86-edb04be1264c/crucible/regions/fe2e91d3-eafe-44ee-bcd3-ec2a10dd793f","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.352246297Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"datafile"}
{"msg":"snapshot is oxp_1b4c5120-bb11-444d-9c86-edb04be1264c/crucible/regions/fe2e91d3-eafe-44ee-bcd3-ec2a10dd793f@a71d0d30-1bc4-4c84-887d-7dc2bb733294","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.360262304Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_1b4c5120-bb11-444d-9c86-edb04be1264c/crucible/regions/fe2e91d3-eafe-44ee-bcd3-ec2a10dd793f\", \"a71d0d30-1bc4-4c84-887d-7dc2bb733294\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.360293895Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot fe2e91d3-eafe-44ee-bcd3-ec2a10dd793f-a71d0d30-1bc4-4c84-887d-7dc2bb733294 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.368566897Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"datafile"}
{"msg":"applying SMF actions for region fe2e91d3-eafe-44ee-bcd3-ec2a10dd793f running snapshot a71d0d30-1bc4-4c84-887d-7dc2bb733294 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.369103848Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.369127798Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"uri":"/crucible/0/regions/fe2e91d3-eafe-44ee-bcd3-ec2a10dd793f/snapshots/a71d0d30-1bc4-4c84-887d-7dc2bb733294/run","method":"POST","req_id":"328fe020-331d-4923-896f-6579bcbac31d","remote_addr":"[fd00:1122:3344:101::a]:61003","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":24958,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-fe2e91d3-eafe-44ee-bcd3-ec2a10dd793f-a71d0d30-1bc4-4c84-887d-7dc2bb733294","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.377844649Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-fe2e91d3-eafe-44ee-bcd3-ec2a10dd793f-a71d0d30-1bc4-4c84-887d-7dc2bb733294","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.382222855Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-fe2e91d3-eafe-44ee-bcd3-ec2a10dd793f-a71d0d30-1bc4-4c84-887d-7dc2bb733294","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.395515736Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/fe2e91d3-eafe-44ee-bcd3-ec2a10dd793f/.zfs/snapshot/a71d0d30-1bc4-4c84-887d-7dc2bb733294","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.395769891Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.395829822Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.395870483Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.395909254Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.395966645Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.39928007Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.404116735Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"worker"}
{"msg":"region fe2e91d3-eafe-44ee-bcd3-ec2a10dd793f running snapshot a71d0d30-1bc4-4c84-887d-7dc2bb733294 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.404128665Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"datafile"}
{"msg":"path is \"/data/regions/fe2e91d3-eafe-44ee-bcd3-ec2a10dd793f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.735842253Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"datafile"}
{"msg":"dataset is oxp_1b4c5120-bb11-444d-9c86-edb04be1264c/crucible/regions/fe2e91d3-eafe-44ee-bcd3-ec2a10dd793f","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.744148416Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"datafile"}
{"msg":"snapshot is oxp_1b4c5120-bb11-444d-9c86-edb04be1264c/crucible/regions/fe2e91d3-eafe-44ee-bcd3-ec2a10dd793f@a71d0d30-1bc4-4c84-887d-7dc2bb733294","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.751188114Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_1b4c5120-bb11-444d-9c86-edb04be1264c/crucible/regions/fe2e91d3-eafe-44ee-bcd3-ec2a10dd793f\", \"a71d0d30-1bc4-4c84-887d-7dc2bb733294\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.751224305Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:31.758989147Z","hostname":"oxz_crucible_3a33ba47-8706-4469-959c-acd4a6a8404e","pid":28481,"uri":"/crucible/0/regions/fe2e91d3-eafe-44ee-bcd3-ec2a10dd793f/snapshots/a71d0d30-1bc4-4c84-887d-7dc2bb733294/run","method":"POST","req_id":"30d02aed-4c45-4ed4-aab7-f9d4932971ed","remote_addr":"[fd00:1122:3344:101::a]:61003","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":23175,"response_code":"200"}