{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:27.519961439Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:46645"} {"msg":"region 05b4d890-85ec-4d93-b935-560599f00d01 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:27.520028661Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:27.521429219Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"uri":"/crucible/0/regions","method":"POST","req_id":"fcf88a69-2900-42a2-b2c4-afafb417d670","remote_addr":"[fd00:1122:3344:101::b]:46645","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":1431,"response_code":200} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:27.52144629Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_b5c1611c-99f5-4a85-acc0-6baa14c5138e/crucible/regions/05b4d890-85ec-4d93-b935-560599f00d01","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:27.528657699Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_b5c1611c-99f5-4a85-acc0-6baa14c5138e/crucible/regions/05b4d890-85ec-4d93-b935-560599f00d01","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:27.528674249Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"05b4d890-85ec-4d93-b935-560599f00d01\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/05b4d890-85ec-4d93-b935-560599f00d01\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:27.580830877Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"region":"05b4d890-85ec-4d93-b935-560599f00d01","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"05b4d890-85ec-4d93-b935-560599f00d01\" \"--data\" \"/data/regions/05b4d890-85ec-4d93-b935-560599f00d01\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:27.580859968Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"region":"05b4d890-85ec-4d93-b935-560599f00d01","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:27.670166111Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"region":"05b4d890-85ec-4d93-b935-560599f00d01","component":"worker"} {"msg":"region 05b4d890-85ec-4d93-b935-560599f00d01 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:27.670206132Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:27.670324975Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-05b4d890-85ec-4d93-b935-560599f00d01","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:27.670793288Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-05b4d890-85ec-4d93-b935-560599f00d01","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:27.682134621Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:27.709205028Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-05b4d890-85ec-4d93-b935-560599f00d01","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:27.713422914Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/05b4d890-85ec-4d93-b935-560599f00d01","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:27.713711772Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:27.713782934Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:27.713841225Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:27.713905077Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:27.718274078Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:27.739932945Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:27.804607648Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"uri":"/crucible/0/regions","method":"POST","req_id":"73a7e989-0bd2-483d-912b-b805d1d7072d","remote_addr":"[fd00:1122:3344:101::b]:46645","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":56,"response_code":200} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:29.604512362Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"uri":"/crucible/0/regions/94824d0a-b98d-4927-b397-b7110859a07b","method":"GET","req_id":"0ac9f230-17ff-4c78-a679-004f2279a327","remote_addr":"[fd00:1122:3344:101::b]:46645","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":68,"response_code":200} {"msg":"path is \"/data/regions/94824d0a-b98d-4927-b397-b7110859a07b\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:29.607842274Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"component":"datafile"} {"msg":"dataset is oxp_b5c1611c-99f5-4a85-acc0-6baa14c5138e/crucible/regions/94824d0a-b98d-4927-b397-b7110859a07b","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:29.614514908Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"component":"datafile"} {"msg":"snapshot is oxp_b5c1611c-99f5-4a85-acc0-6baa14c5138e/crucible/regions/94824d0a-b98d-4927-b397-b7110859a07b@c8a59639-5d7e-4a5a-b19d-b92aa18f6ead","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:29.620705999Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_b5c1611c-99f5-4a85-acc0-6baa14c5138e/crucible/regions/94824d0a-b98d-4927-b397-b7110859a07b\", \"c8a59639-5d7e-4a5a-b19d-b92aa18f6ead\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:29.620716889Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:29.628398831Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"uri":"/crucible/0/regions/94824d0a-b98d-4927-b397-b7110859a07b/snapshots/c8a59639-5d7e-4a5a-b19d-b92aa18f6ead","method":"GET","req_id":"d7267604-315f-417d-a019-438f7000d6f3","remote_addr":"[fd00:1122:3344:101::b]:46645","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":20576,"response_code":200} {"msg":"path is \"/data/regions/94824d0a-b98d-4927-b397-b7110859a07b\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:29.631972369Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"component":"datafile"} {"msg":"dataset is oxp_b5c1611c-99f5-4a85-acc0-6baa14c5138e/crucible/regions/94824d0a-b98d-4927-b397-b7110859a07b","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:29.638671844Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"component":"datafile"} {"msg":"snapshot is oxp_b5c1611c-99f5-4a85-acc0-6baa14c5138e/crucible/regions/94824d0a-b98d-4927-b397-b7110859a07b@c8a59639-5d7e-4a5a-b19d-b92aa18f6ead","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:29.644453303Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_b5c1611c-99f5-4a85-acc0-6baa14c5138e/crucible/regions/94824d0a-b98d-4927-b397-b7110859a07b\", \"c8a59639-5d7e-4a5a-b19d-b92aa18f6ead\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:29.644473484Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot 94824d0a-b98d-4927-b397-b7110859a07b-c8a59639-5d7e-4a5a-b19d-b92aa18f6ead state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:29.651568168Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:29.651684478Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"uri":"/crucible/0/regions/94824d0a-b98d-4927-b397-b7110859a07b/snapshots/c8a59639-5d7e-4a5a-b19d-b92aa18f6ead/run","method":"POST","req_id":"41d9eef5-2118-4cda-82a3-9c9fd91b16d1","remote_addr":"[fd00:1122:3344:101::b]:46645","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":19754,"response_code":200} {"msg":"applying SMF actions for region 94824d0a-b98d-4927-b397-b7110859a07b running snapshot c8a59639-5d7e-4a5a-b19d-b92aa18f6ead (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:29.65170065Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"component":"worker"} {"msg":"creating missing snapshot instance snapshot-94824d0a-b98d-4927-b397-b7110859a07b-c8a59639-5d7e-4a5a-b19d-b92aa18f6ead","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:29.656355012Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-94824d0a-b98d-4927-b397-b7110859a07b-c8a59639-5d7e-4a5a-b19d-b92aa18f6ead","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:29.664186368Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-94824d0a-b98d-4927-b397-b7110859a07b-c8a59639-5d7e-4a5a-b19d-b92aa18f6ead","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:29.675491019Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/94824d0a-b98d-4927-b397-b7110859a07b/.zfs/snapshot/c8a59639-5d7e-4a5a-b19d-b92aa18f6ead","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:29.675718206Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:29.675760927Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:29.675802628Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:29.675837469Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:29.6758796Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:29.679442218Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:29.684096657Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"component":"worker"} {"msg":"region 94824d0a-b98d-4927-b397-b7110859a07b running snapshot c8a59639-5d7e-4a5a-b19d-b92aa18f6ead state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:29.684107427Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"component":"datafile"} {"msg":"path is \"/data/regions/94824d0a-b98d-4927-b397-b7110859a07b\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:30.022968642Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"component":"datafile"} {"msg":"dataset is oxp_b5c1611c-99f5-4a85-acc0-6baa14c5138e/crucible/regions/94824d0a-b98d-4927-b397-b7110859a07b","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:30.032772472Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"component":"datafile"} {"msg":"snapshot is oxp_b5c1611c-99f5-4a85-acc0-6baa14c5138e/crucible/regions/94824d0a-b98d-4927-b397-b7110859a07b@c8a59639-5d7e-4a5a-b19d-b92aa18f6ead","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:30.04247275Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_b5c1611c-99f5-4a85-acc0-6baa14c5138e/crucible/regions/94824d0a-b98d-4927-b397-b7110859a07b\", \"c8a59639-5d7e-4a5a-b19d-b92aa18f6ead\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:30.042524711Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T05:54:30.05083465Z","hostname":"oxz_crucible_9d0312de-0728-44b8-bc8f-be40888294e2","pid":23767,"uri":"/crucible/0/regions/94824d0a-b98d-4927-b397-b7110859a07b/snapshots/c8a59639-5d7e-4a5a-b19d-b92aa18f6ead/run","method":"POST","req_id":"9da72538-a7bd-4946-b0a0-dc25a97cc116","remote_addr":"[fd00:1122:3344:101::b]:46645","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":27921,"response_code":200}