{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:27.568708884Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:53512"} {"msg":"region a731c5cf-bccc-444d-9a0b-1ed9b17ada88 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:27.617117107Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:27.624826413Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:27.624854493Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"uri":"/crucible/0/regions","method":"POST","req_id":"19e3b6b2-5708-4835-99eb-d2e2bcfef6f9","remote_addr":"[fd00:1122:3344:101::b]:53512","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":26521,"response_code":200} {"msg":"zfs set reservation of 2684354560 for oxp_f4ef9696-c034-4f43-8203-40e43adf81c3/crucible/regions/a731c5cf-bccc-444d-9a0b-1ed9b17ada88","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:27.635424549Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_f4ef9696-c034-4f43-8203-40e43adf81c3/crucible/regions/a731c5cf-bccc-444d-9a0b-1ed9b17ada88","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:27.63544959Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"a731c5cf-bccc-444d-9a0b-1ed9b17ada88\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/a731c5cf-bccc-444d-9a0b-1ed9b17ada88\"","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:27.689289809Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"region":"a731c5cf-bccc-444d-9a0b-1ed9b17ada88","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"a731c5cf-bccc-444d-9a0b-1ed9b17ada88\" \"--data\" \"/data/regions/a731c5cf-bccc-444d-9a0b-1ed9b17ada88\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:27.691489878Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"region":"a731c5cf-bccc-444d-9a0b-1ed9b17ada88","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:27.973283494Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"uri":"/crucible/0/regions","method":"POST","req_id":"76aecc52-2c84-43dc-9e16-574b74e4e7a5","remote_addr":"[fd00:1122:3344:101::b]:53512","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":60,"response_code":200} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:27.993253716Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"region":"a731c5cf-bccc-444d-9a0b-1ed9b17ada88","component":"worker"} {"msg":"region a731c5cf-bccc-444d-9a0b-1ed9b17ada88 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:27.993270288Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:27.993385865Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-a731c5cf-bccc-444d-9a0b-1ed9b17ada88","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:27.993750953Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-a731c5cf-bccc-444d-9a0b-1ed9b17ada88","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:28.005340879Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:28.027727914Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a731c5cf-bccc-444d-9a0b-1ed9b17ada88","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:28.03093106Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a731c5cf-bccc-444d-9a0b-1ed9b17ada88","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:28.031635473Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:28.031693974Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:28.031730645Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:28.031763625Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:28.035538622Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:28.039348639Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:28.675593113Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"uri":"/crucible/0/regions","method":"POST","req_id":"dbec2e2a-1089-4107-8513-f2a1e34f03aa","remote_addr":"[fd00:1122:3344:101::b]:53512","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":61,"response_code":200} {"msg":"region 33d634fd-29b2-48d7-a9aa-8578397ddbcb state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:34.584811708Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:34.584977351Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"uri":"/crucible/0/regions","method":"POST","req_id":"c2473c6b-3beb-420c-b566-b293f1fda1ed","remote_addr":"[fd00:1122:3344:101::b]:53512","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":224,"response_code":200} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:34.584993761Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_f4ef9696-c034-4f43-8203-40e43adf81c3/crucible/regions/33d634fd-29b2-48d7-a9aa-8578397ddbcb","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:34.591550127Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_f4ef9696-c034-4f43-8203-40e43adf81c3/crucible/regions/33d634fd-29b2-48d7-a9aa-8578397ddbcb","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:34.591569307Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"33d634fd-29b2-48d7-a9aa-8578397ddbcb\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/33d634fd-29b2-48d7-a9aa-8578397ddbcb\"","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:34.6467733Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"region":"33d634fd-29b2-48d7-a9aa-8578397ddbcb","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"33d634fd-29b2-48d7-a9aa-8578397ddbcb\" \"--data\" \"/data/regions/33d634fd-29b2-48d7-a9aa-8578397ddbcb\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:34.646811271Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"region":"33d634fd-29b2-48d7-a9aa-8578397ddbcb","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:34.721126681Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"region":"33d634fd-29b2-48d7-a9aa-8578397ddbcb","component":"worker"} {"msg":"region 33d634fd-29b2-48d7-a9aa-8578397ddbcb state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:34.721158401Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:34.721248153Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-33d634fd-29b2-48d7-a9aa-8578397ddbcb","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:34.721575459Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-33d634fd-29b2-48d7-a9aa-8578397ddbcb","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:34.724996749Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:34.731738538Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-33d634fd-29b2-48d7-a9aa-8578397ddbcb","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:34.734538877Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/33d634fd-29b2-48d7-a9aa-8578397ddbcb","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:34.734831412Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:34.734875163Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:34.734913284Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:34.734950334Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:34.73865545Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:34.744030455Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:16:34.768873352Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"uri":"/crucible/0/regions","method":"POST","req_id":"04940f6e-6818-4139-b102-e225f3f3ecd1","remote_addr":"[fd00:1122:3344:101::b]:53512","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":57,"response_code":200} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:17:02.434997011Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"uri":"/crucible/0/regions/33d634fd-29b2-48d7-a9aa-8578397ddbcb","method":"GET","req_id":"31d9dfd9-1228-4ac4-85ae-744e5c1e9ac6","remote_addr":"[fd00:1122:3344:101::b]:53512","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":89,"response_code":200} {"msg":"path is \"/data/regions/33d634fd-29b2-48d7-a9aa-8578397ddbcb\"","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:17:02.439077582Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"component":"datafile"} {"msg":"dataset is oxp_f4ef9696-c034-4f43-8203-40e43adf81c3/crucible/regions/33d634fd-29b2-48d7-a9aa-8578397ddbcb","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:17:02.458284631Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"component":"datafile"} {"msg":"region 33d634fd-29b2-48d7-a9aa-8578397ddbcb state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:17:02.464524341Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:17:02.464669714Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"uri":"/crucible/0/regions/33d634fd-29b2-48d7-a9aa-8578397ddbcb","method":"DELETE","req_id":"1bddfec3-bc81-4b11-b872-d4488df68327","remote_addr":"[fd00:1122:3344:101::b]:53512","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":25623,"response_code":204} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:17:02.464692364Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"component":"worker"} {"msg":"disabling downstairs instance: downstairs-33d634fd-29b2-48d7-a9aa-8578397ddbcb (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:17:02.46559227Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:17:02.467922641Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"uri":"/crucible/0/regions/33d634fd-29b2-48d7-a9aa-8578397ddbcb","method":"GET","req_id":"7cd78311-c01d-4678-a93e-b6bcb5362bd3","remote_addr":"[fd00:1122:3344:101::b]:53512","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":33,"response_code":200} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:17:02.491370064Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_f4ef9696-c034-4f43-8203-40e43adf81c3/crucible/regions/33d634fd-29b2-48d7-a9aa-8578397ddbcb\"","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:17:02.497841448Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"region":"33d634fd-29b2-48d7-a9aa-8578397ddbcb","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:17:02.555369472Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"uri":"/crucible/0/regions/33d634fd-29b2-48d7-a9aa-8578397ddbcb","method":"GET","req_id":"cb7863a6-e85f-41d9-9845-f8751d5a9001","remote_addr":"[fd00:1122:3344:101::b]:53512","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":52,"response_code":200} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:17:02.627216188Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"uri":"/crucible/0/regions/33d634fd-29b2-48d7-a9aa-8578397ddbcb","method":"GET","req_id":"0b0fab30-58b1-4858-a0c7-9bcdab0a4e60","remote_addr":"[fd00:1122:3344:101::b]:53512","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":35,"response_code":200} {"msg":"region 33d634fd-29b2-48d7-a9aa-8578397ddbcb state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:17:02.648735008Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T23:17:02.811031538Z","hostname":"oxz_crucible_94706b58-b2e8-4408-826f-f825e7ac7f8e","pid":23925,"uri":"/crucible/0/regions/33d634fd-29b2-48d7-a9aa-8578397ddbcb","method":"GET","req_id":"449a7040-81c8-433a-bb59-3bbb6e141de2","remote_addr":"[fd00:1122:3344:101::b]:53512","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":29,"response_code":200}