{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:40.169089834Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:34525"} {"msg":"region abda1616-1e3c-4df4-9011-8ff685987e96 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:40.169198516Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:40.16989022Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"uri":"/crucible/0/regions","method":"POST","req_id":"c5afff63-a1cd-49fc-928b-04c61308f303","remote_addr":"[fd00:1122:3344:101::b]:34525","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":692,"response_code":200} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:40.169912211Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_ffc24c46-2e65-4754-8308-75c016d91387/crucible/regions/abda1616-1e3c-4df4-9011-8ff685987e96","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:40.177455398Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_ffc24c46-2e65-4754-8308-75c016d91387/crucible/regions/abda1616-1e3c-4df4-9011-8ff685987e96","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:40.177475638Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"abda1616-1e3c-4df4-9011-8ff685987e96\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/abda1616-1e3c-4df4-9011-8ff685987e96\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:40.226031408Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"region":"abda1616-1e3c-4df4-9011-8ff685987e96","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"abda1616-1e3c-4df4-9011-8ff685987e96\" \"--data\" \"/data/regions/abda1616-1e3c-4df4-9011-8ff685987e96\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:40.226064639Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"region":"abda1616-1e3c-4df4-9011-8ff685987e96","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:40.345037024Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"region":"abda1616-1e3c-4df4-9011-8ff685987e96","component":"worker"} {"msg":"region abda1616-1e3c-4df4-9011-8ff685987e96 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:40.345078035Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:40.345174487Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-abda1616-1e3c-4df4-9011-8ff685987e96","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:40.365789606Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-abda1616-1e3c-4df4-9011-8ff685987e96","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:40.372084727Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:40.383654858Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-abda1616-1e3c-4df4-9011-8ff685987e96","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:40.386469736Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/abda1616-1e3c-4df4-9011-8ff685987e96","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:40.386761172Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:40.386802493Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:40.386840574Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:40.386874635Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:40.390435839Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:40.395223939Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:40.419027024Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"uri":"/crucible/0/regions","method":"POST","req_id":"1b468419-756c-4003-a1bc-6fa50fe20ac4","remote_addr":"[fd00:1122:3344:101::b]:34525","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":63,"response_code":200} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:41.706474517Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"uri":"/crucible/0/regions/3a067ee1-29d5-4835-b2ca-390e46056fe2","method":"GET","req_id":"a3de44b7-429b-40af-89f5-1e0fd0fb5b0b","remote_addr":"[fd00:1122:3344:101::b]:34525","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":73,"response_code":200} {"msg":"path is \"/data/regions/3a067ee1-29d5-4835-b2ca-390e46056fe2\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:41.709468963Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"component":"datafile"} {"msg":"dataset is oxp_ffc24c46-2e65-4754-8308-75c016d91387/crucible/regions/3a067ee1-29d5-4835-b2ca-390e46056fe2","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:41.715853492Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"component":"datafile"} {"msg":"snapshot is oxp_ffc24c46-2e65-4754-8308-75c016d91387/crucible/regions/3a067ee1-29d5-4835-b2ca-390e46056fe2@f1a02dc4-ab12-4949-8791-fda9f0d52fc6","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:41.722702634Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_ffc24c46-2e65-4754-8308-75c016d91387/crucible/regions/3a067ee1-29d5-4835-b2ca-390e46056fe2\", \"f1a02dc4-ab12-4949-8791-fda9f0d52fc6\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:41.722717944Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:41.731284513Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"uri":"/crucible/0/regions/3a067ee1-29d5-4835-b2ca-390e46056fe2/snapshots/f1a02dc4-ab12-4949-8791-fda9f0d52fc6","method":"GET","req_id":"dee94cf5-bca3-441d-b8bb-d7aade4325c2","remote_addr":"[fd00:1122:3344:101::b]:34525","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":21785,"response_code":200} {"msg":"path is \"/data/regions/3a067ee1-29d5-4835-b2ca-390e46056fe2\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:41.734676413Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"component":"datafile"} {"msg":"dataset is oxp_ffc24c46-2e65-4754-8308-75c016d91387/crucible/regions/3a067ee1-29d5-4835-b2ca-390e46056fe2","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:41.740102696Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"component":"datafile"} {"msg":"snapshot is oxp_ffc24c46-2e65-4754-8308-75c016d91387/crucible/regions/3a067ee1-29d5-4835-b2ca-390e46056fe2@f1a02dc4-ab12-4949-8791-fda9f0d52fc6","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:41.744894446Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_ffc24c46-2e65-4754-8308-75c016d91387/crucible/regions/3a067ee1-29d5-4835-b2ca-390e46056fe2\", \"f1a02dc4-ab12-4949-8791-fda9f0d52fc6\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:41.744911956Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot 3a067ee1-29d5-4835-b2ca-390e46056fe2-f1a02dc4-ab12-4949-8791-fda9f0d52fc6 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:41.753037205Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:41.753172488Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"uri":"/crucible/0/regions/3a067ee1-29d5-4835-b2ca-390e46056fe2/snapshots/f1a02dc4-ab12-4949-8791-fda9f0d52fc6/run","method":"POST","req_id":"7c69d52a-8e73-4f42-bba8-9dd4a633aaff","remote_addr":"[fd00:1122:3344:101::b]:34525","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":18513,"response_code":200} {"msg":"applying SMF actions for region 3a067ee1-29d5-4835-b2ca-390e46056fe2 running snapshot f1a02dc4-ab12-4949-8791-fda9f0d52fc6 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:41.753189649Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"component":"worker"} {"msg":"creating missing snapshot instance snapshot-3a067ee1-29d5-4835-b2ca-390e46056fe2-f1a02dc4-ab12-4949-8791-fda9f0d52fc6","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:41.757333095Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-3a067ee1-29d5-4835-b2ca-390e46056fe2-f1a02dc4-ab12-4949-8791-fda9f0d52fc6","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:41.76958695Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-3a067ee1-29d5-4835-b2ca-390e46056fe2-f1a02dc4-ab12-4949-8791-fda9f0d52fc6","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:41.779247221Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3a067ee1-29d5-4835-b2ca-390e46056fe2/.zfs/snapshot/f1a02dc4-ab12-4949-8791-fda9f0d52fc6","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:41.779516196Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:41.779555897Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:41.779591108Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:41.779625359Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:41.779666579Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:41.782952938Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:41.787116694Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"component":"worker"} {"msg":"region 3a067ee1-29d5-4835-b2ca-390e46056fe2 running snapshot f1a02dc4-ab12-4949-8791-fda9f0d52fc6 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:41.787125344Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"component":"datafile"} {"msg":"path is \"/data/regions/3a067ee1-29d5-4835-b2ca-390e46056fe2\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:41.992316173Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"component":"datafile"} {"msg":"dataset is oxp_ffc24c46-2e65-4754-8308-75c016d91387/crucible/regions/3a067ee1-29d5-4835-b2ca-390e46056fe2","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:41.998768668Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"component":"datafile"} {"msg":"snapshot is oxp_ffc24c46-2e65-4754-8308-75c016d91387/crucible/regions/3a067ee1-29d5-4835-b2ca-390e46056fe2@f1a02dc4-ab12-4949-8791-fda9f0d52fc6","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:42.00464325Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_ffc24c46-2e65-4754-8308-75c016d91387/crucible/regions/3a067ee1-29d5-4835-b2ca-390e46056fe2\", \"f1a02dc4-ab12-4949-8791-fda9f0d52fc6\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:42.00465655Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-17T00:00:42.010770817Z","hostname":"oxz_crucible_a8177aca-7d18-48ff-97fa-a148be6762e3","pid":23737,"uri":"/crucible/0/regions/3a067ee1-29d5-4835-b2ca-390e46056fe2/snapshots/f1a02dc4-ab12-4949-8791-fda9f0d52fc6/run","method":"POST","req_id":"b000147b-910b-453a-804d-4e679092c782","remote_addr":"[fd00:1122:3344:101::b]:34525","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":18494,"response_code":200}