{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:36.767264407Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:34357"} {"msg":"region eab25e16-fc1e-4e82-be1b-5294c2875638 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:36.7673895Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:36.768050183Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:36.768060493Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"uri":"/crucible/0/regions","method":"POST","req_id":"b8fdd0c0-e1a1-4190-9316-2a035499c44a","remote_addr":"[fd00:1122:3344:101::b]:34357","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":705,"response_code":200} {"msg":"zfs set reservation of 2684354560 for oxp_c3b318a9-f2b9-4030-8b3b-6276c9fc6736/crucible/regions/eab25e16-fc1e-4e82-be1b-5294c2875638","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:36.776280897Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_c3b318a9-f2b9-4030-8b3b-6276c9fc6736/crucible/regions/eab25e16-fc1e-4e82-be1b-5294c2875638","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:36.776300577Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"eab25e16-fc1e-4e82-be1b-5294c2875638\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/eab25e16-fc1e-4e82-be1b-5294c2875638\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:36.831746619Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"region":"eab25e16-fc1e-4e82-be1b-5294c2875638","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"eab25e16-fc1e-4e82-be1b-5294c2875638\" \"--data\" \"/data/regions/eab25e16-fc1e-4e82-be1b-5294c2875638\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:36.831777189Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"region":"eab25e16-fc1e-4e82-be1b-5294c2875638","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:36.949956239Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"region":"eab25e16-fc1e-4e82-be1b-5294c2875638","component":"worker"} {"msg":"region eab25e16-fc1e-4e82-be1b-5294c2875638 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:36.949993709Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:36.950112582Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-eab25e16-fc1e-4e82-be1b-5294c2875638","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:36.970476186Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-eab25e16-fc1e-4e82-be1b-5294c2875638","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:36.974141929Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:36.98321548Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-eab25e16-fc1e-4e82-be1b-5294c2875638","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:36.98572519Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/eab25e16-fc1e-4e82-be1b-5294c2875638","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:36.986057806Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:36.986099337Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:36.986136768Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:36.986174298Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:36.989386632Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:36.994227429Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:37.094081313Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"uri":"/crucible/0/regions","method":"POST","req_id":"48289611-6dec-4bea-9692-d4f11b3ca9be","remote_addr":"[fd00:1122:3344:101::b]:34357","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":68,"response_code":200} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:37.896486813Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"uri":"/crucible/0/regions/7a5c0599-72eb-4f87-9d18-3b83567c584d","method":"GET","req_id":"72f10383-faa5-4b6f-8cba-03951794a06d","remote_addr":"[fd00:1122:3344:101::b]:34357","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":72,"response_code":200} {"msg":"path is \"/data/regions/7a5c0599-72eb-4f87-9d18-3b83567c584d\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:37.902333129Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"component":"datafile"} {"msg":"dataset is oxp_c3b318a9-f2b9-4030-8b3b-6276c9fc6736/crucible/regions/7a5c0599-72eb-4f87-9d18-3b83567c584d","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:37.908859289Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"component":"datafile"} {"msg":"snapshot is oxp_c3b318a9-f2b9-4030-8b3b-6276c9fc6736/crucible/regions/7a5c0599-72eb-4f87-9d18-3b83567c584d@d52c6aa1-2bc0-4ba8-9230-9436d57dbe56","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:37.916369318Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_c3b318a9-f2b9-4030-8b3b-6276c9fc6736/crucible/regions/7a5c0599-72eb-4f87-9d18-3b83567c584d\", \"d52c6aa1-2bc0-4ba8-9230-9436d57dbe56\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:37.916386168Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:37.925352126Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"uri":"/crucible/0/regions/7a5c0599-72eb-4f87-9d18-3b83567c584d/snapshots/d52c6aa1-2bc0-4ba8-9230-9436d57dbe56","method":"GET","req_id":"b61fe3b1-4330-40a7-a680-3f50fa3678c6","remote_addr":"[fd00:1122:3344:101::b]:34357","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":23047,"response_code":200} {"msg":"path is \"/data/regions/7a5c0599-72eb-4f87-9d18-3b83567c584d\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:37.931433607Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"component":"datafile"} {"msg":"dataset is oxp_c3b318a9-f2b9-4030-8b3b-6276c9fc6736/crucible/regions/7a5c0599-72eb-4f87-9d18-3b83567c584d","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:37.93708331Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"component":"datafile"} {"msg":"snapshot is oxp_c3b318a9-f2b9-4030-8b3b-6276c9fc6736/crucible/regions/7a5c0599-72eb-4f87-9d18-3b83567c584d@d52c6aa1-2bc0-4ba8-9230-9436d57dbe56","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:37.943356814Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_c3b318a9-f2b9-4030-8b3b-6276c9fc6736/crucible/regions/7a5c0599-72eb-4f87-9d18-3b83567c584d\", \"d52c6aa1-2bc0-4ba8-9230-9436d57dbe56\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:37.943374915Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot 7a5c0599-72eb-4f87-9d18-3b83567c584d-d52c6aa1-2bc0-4ba8-9230-9436d57dbe56 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:37.949614349Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"component":"datafile"} {"msg":"applying SMF actions for region 7a5c0599-72eb-4f87-9d18-3b83567c584d running snapshot d52c6aa1-2bc0-4ba8-9230-9436d57dbe56 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:37.949751471Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:37.949766072Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"uri":"/crucible/0/regions/7a5c0599-72eb-4f87-9d18-3b83567c584d/snapshots/d52c6aa1-2bc0-4ba8-9230-9436d57dbe56/run","method":"POST","req_id":"6cd81192-431c-48de-a560-ad3e6673f58a","remote_addr":"[fd00:1122:3344:101::b]:34357","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":18341,"response_code":200} {"msg":"creating missing snapshot instance snapshot-7a5c0599-72eb-4f87-9d18-3b83567c584d-d52c6aa1-2bc0-4ba8-9230-9436d57dbe56","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:37.954383574Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-7a5c0599-72eb-4f87-9d18-3b83567c584d-d52c6aa1-2bc0-4ba8-9230-9436d57dbe56","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:37.962208829Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-7a5c0599-72eb-4f87-9d18-3b83567c584d-d52c6aa1-2bc0-4ba8-9230-9436d57dbe56","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:37.971748609Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/7a5c0599-72eb-4f87-9d18-3b83567c584d/.zfs/snapshot/d52c6aa1-2bc0-4ba8-9230-9436d57dbe56","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:37.971946883Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:37.971986153Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:37.972024674Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:37.972057595Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:37.972101226Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:37.975713667Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:37.980087054Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"component":"worker"} {"msg":"region 7a5c0599-72eb-4f87-9d18-3b83567c584d running snapshot d52c6aa1-2bc0-4ba8-9230-9436d57dbe56 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:37.980096905Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"component":"datafile"} {"msg":"path is \"/data/regions/7a5c0599-72eb-4f87-9d18-3b83567c584d\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:38.328344656Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"component":"datafile"} {"msg":"dataset is oxp_c3b318a9-f2b9-4030-8b3b-6276c9fc6736/crucible/regions/7a5c0599-72eb-4f87-9d18-3b83567c584d","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:38.334278514Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"component":"datafile"} {"msg":"snapshot is oxp_c3b318a9-f2b9-4030-8b3b-6276c9fc6736/crucible/regions/7a5c0599-72eb-4f87-9d18-3b83567c584d@d52c6aa1-2bc0-4ba8-9230-9436d57dbe56","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:38.339286113Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_c3b318a9-f2b9-4030-8b3b-6276c9fc6736/crucible/regions/7a5c0599-72eb-4f87-9d18-3b83567c584d\", \"d52c6aa1-2bc0-4ba8-9230-9436d57dbe56\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:38.339309004Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-06T06:03:38.345424625Z","hostname":"oxz_crucible_93cee26a-512b-4901-9337-28dfa9c46f92","pid":23839,"uri":"/crucible/0/regions/7a5c0599-72eb-4f87-9d18-3b83567c584d/snapshots/d52c6aa1-2bc0-4ba8-9230-9436d57dbe56/run","method":"POST","req_id":"a313402d-b4ce-4a7a-b4b8-f73f9f241715","remote_addr":"[fd00:1122:3344:101::b]:34357","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":17129,"response_code":200}