[ Oct 28 02:47:37 Enabled. ]
[ Oct 28 02:47:37 Rereading configuration. ]
[ Oct 28 02:47:41 Rereading configuration. ]
[ Oct 28 02:47:48 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_7260b4b3-a573-4f9c-9e86-8952e72fc71e/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 75ea3408643664877aa603ffc65996ed6dc34c0a\nCommit timestamp: 2025-10-24T15:21:57.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:47:48.286611969Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403}
{"msg":"dataset: \"oxp_7260b4b3-a573-4f9c-9e86-8952e72fc71e/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:47:48.286827146Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403}
{"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:47:48.286838886Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:47:48.286845387Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:47:48.303983017Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:47:48.431388705Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:15.709596429Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:55958"}
{"msg":"region c715eb89-86a6-4399-bfcf-83b55f3f70bb state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:15.741691629Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:15.744323025Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"uri":"/crucible/0/regions","method":"POST","req_id":"75513a36-6967-4bdf-a1ab-eb53190cd786","remote_addr":"[fd00:1122:3344:101::a]:55958","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":15423,"response_code":200}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:15.744372766Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_7260b4b3-a573-4f9c-9e86-8952e72fc71e/crucible/regions/c715eb89-86a6-4399-bfcf-83b55f3f70bb","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:15.75150794Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_7260b4b3-a573-4f9c-9e86-8952e72fc71e/crucible/regions/c715eb89-86a6-4399-bfcf-83b55f3f70bb","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:15.75152942Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"c715eb89-86a6-4399-bfcf-83b55f3f70bb\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/c715eb89-86a6-4399-bfcf-83b55f3f70bb\"","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:15.808895477Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"region":"c715eb89-86a6-4399-bfcf-83b55f3f70bb","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"c715eb89-86a6-4399-bfcf-83b55f3f70bb\" \"--data\" \"/data/regions/c715eb89-86a6-4399-bfcf-83b55f3f70bb\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:15.811154201Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"region":"c715eb89-86a6-4399-bfcf-83b55f3f70bb","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:16.103002737Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"uri":"/crucible/0/regions","method":"POST","req_id":"48300229-c9b2-43c7-be75-df59ee6a5666","remote_addr":"[fd00:1122:3344:101::a]:55958","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":61,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:16.12417966Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"region":"c715eb89-86a6-4399-bfcf-83b55f3f70bb","component":"worker"}
{"msg":"region c715eb89-86a6-4399-bfcf-83b55f3f70bb state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:16.124233622Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:16.124433848Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-c715eb89-86a6-4399-bfcf-83b55f3f70bb","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:16.124818791Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-c715eb89-86a6-4399-bfcf-83b55f3f70bb","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:16.131595683Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:16.148616809Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-c715eb89-86a6-4399-bfcf-83b55f3f70bb","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:16.15170264Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c715eb89-86a6-4399-bfcf-83b55f3f70bb","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:16.152404773Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:16.152472615Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:16.152519187Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:16.152552378Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:16.156866419Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:16.161262873Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:48:16.582607725Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"uri":"/crucible/0/regions","method":"POST","req_id":"a2d7828a-a677-46d6-b231-ca1844374e6b","remote_addr":"[fd00:1122:3344:101::a]:55958","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":58,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:48.147555413Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:61183"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:48.150917914Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"uri":"/crucible/0/regions/c715eb89-86a6-4399-bfcf-83b55f3f70bb","method":"GET","req_id":"061db0a3-565b-4a4a-8c93-f1fe43b6a91a","remote_addr":"[fd00:1122:3344:101::a]:61183","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":3331,"response_code":200}
{"msg":"path is \"/data/regions/c715eb89-86a6-4399-bfcf-83b55f3f70bb\"","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:48.154813411Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"datafile"}
{"msg":"dataset is oxp_7260b4b3-a573-4f9c-9e86-8952e72fc71e/crucible/regions/c715eb89-86a6-4399-bfcf-83b55f3f70bb","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:48.162271365Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"datafile"}
{"msg":"snapshot is oxp_7260b4b3-a573-4f9c-9e86-8952e72fc71e/crucible/regions/c715eb89-86a6-4399-bfcf-83b55f3f70bb@8fc0ae4b-c3c0-407e-9ad1-359e52b18c9a","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:48.168620413Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_7260b4b3-a573-4f9c-9e86-8952e72fc71e/crucible/regions/c715eb89-86a6-4399-bfcf-83b55f3f70bb\", \"8fc0ae4b-c3c0-407e-9ad1-359e52b18c9a\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:48.168650214Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:48.17954079Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"uri":"/crucible/0/regions/c715eb89-86a6-4399-bfcf-83b55f3f70bb/snapshots/8fc0ae4b-c3c0-407e-9ad1-359e52b18c9a","method":"GET","req_id":"bc535451-9d8b-4a88-95af-fc497204bc5c","remote_addr":"[fd00:1122:3344:101::a]:61183","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":24751,"response_code":200}
{"msg":"path is \"/data/regions/c715eb89-86a6-4399-bfcf-83b55f3f70bb\"","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:48.183685385Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"datafile"}
{"msg":"dataset is oxp_7260b4b3-a573-4f9c-9e86-8952e72fc71e/crucible/regions/c715eb89-86a6-4399-bfcf-83b55f3f70bb","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:48.191390447Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"datafile"}
{"msg":"snapshot is oxp_7260b4b3-a573-4f9c-9e86-8952e72fc71e/crucible/regions/c715eb89-86a6-4399-bfcf-83b55f3f70bb@8fc0ae4b-c3c0-407e-9ad1-359e52b18c9a","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:48.198041055Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_7260b4b3-a573-4f9c-9e86-8952e72fc71e/crucible/regions/c715eb89-86a6-4399-bfcf-83b55f3f70bb\", \"8fc0ae4b-c3c0-407e-9ad1-359e52b18c9a\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:48.198065456Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot c715eb89-86a6-4399-bfcf-83b55f3f70bb-8fc0ae4b-c3c0-407e-9ad1-359e52b18c9a state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:48.206657867Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:48.207701461Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"uri":"/crucible/0/regions/c715eb89-86a6-4399-bfcf-83b55f3f70bb/snapshots/8fc0ae4b-c3c0-407e-9ad1-359e52b18c9a/run","method":"POST","req_id":"36619979-9a56-499e-b1d0-9f317ae2995c","remote_addr":"[fd00:1122:3344:101::a]:61183","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":24049,"response_code":200}
{"msg":"applying SMF actions for region c715eb89-86a6-4399-bfcf-83b55f3f70bb running snapshot 8fc0ae4b-c3c0-407e-9ad1-359e52b18c9a (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:48.207736532Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-c715eb89-86a6-4399-bfcf-83b55f3f70bb-8fc0ae4b-c3c0-407e-9ad1-359e52b18c9a","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:48.225586036Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-c715eb89-86a6-4399-bfcf-83b55f3f70bb-8fc0ae4b-c3c0-407e-9ad1-359e52b18c9a","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:48.237349971Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-c715eb89-86a6-4399-bfcf-83b55f3f70bb-8fc0ae4b-c3c0-407e-9ad1-359e52b18c9a","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:48.249981844Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c715eb89-86a6-4399-bfcf-83b55f3f70bb/.zfs/snapshot/8fc0ae4b-c3c0-407e-9ad1-359e52b18c9a","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:48.250238382Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:48.250289174Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:48.250331415Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:48.250374517Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:48.250437029Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:48.254918195Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:48.259429393Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"region c715eb89-86a6-4399-bfcf-83b55f3f70bb running snapshot 8fc0ae4b-c3c0-407e-9ad1-359e52b18c9a state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:48.259448484Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"datafile"}
{"msg":"path is \"/data/regions/c715eb89-86a6-4399-bfcf-83b55f3f70bb\"","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:48.566955292Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"datafile"}
{"msg":"dataset is oxp_7260b4b3-a573-4f9c-9e86-8952e72fc71e/crucible/regions/c715eb89-86a6-4399-bfcf-83b55f3f70bb","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:48.574404696Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"datafile"}
{"msg":"snapshot is oxp_7260b4b3-a573-4f9c-9e86-8952e72fc71e/crucible/regions/c715eb89-86a6-4399-bfcf-83b55f3f70bb@8fc0ae4b-c3c0-407e-9ad1-359e52b18c9a","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:48.580782334Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_7260b4b3-a573-4f9c-9e86-8952e72fc71e/crucible/regions/c715eb89-86a6-4399-bfcf-83b55f3f70bb\", \"8fc0ae4b-c3c0-407e-9ad1-359e52b18c9a\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:48.580816516Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:48.586216182Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"uri":"/crucible/0/regions/c715eb89-86a6-4399-bfcf-83b55f3f70bb/snapshots/8fc0ae4b-c3c0-407e-9ad1-359e52b18c9a/run","method":"POST","req_id":"77133038-1f4c-4626-b8e6-7279b75579c7","remote_addr":"[fd00:1122:3344:101::a]:61183","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":19300,"response_code":200}
{"msg":"region df08bd09-c1e4-48b1-815c-cb9ad807607d state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:53.17878626Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:53.178975146Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"uri":"/crucible/0/regions","method":"POST","req_id":"1cd44ffd-d846-4718-93fb-ae5ecb5f4107","remote_addr":"[fd00:1122:3344:101::a]:61183","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":243,"response_code":200}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:53.178998747Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_7260b4b3-a573-4f9c-9e86-8952e72fc71e/crucible/regions/df08bd09-c1e4-48b1-815c-cb9ad807607d","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:53.185090306Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_7260b4b3-a573-4f9c-9e86-8952e72fc71e/crucible/regions/df08bd09-c1e4-48b1-815c-cb9ad807607d","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:53.185116197Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"df08bd09-c1e4-48b1-815c-cb9ad807607d\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/df08bd09-c1e4-48b1-815c-cb9ad807607d\"","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:53.239397503Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"region":"df08bd09-c1e4-48b1-815c-cb9ad807607d","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"df08bd09-c1e4-48b1-815c-cb9ad807607d\" \"--data\" \"/data/regions/df08bd09-c1e4-48b1-815c-cb9ad807607d\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:53.239440844Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"region":"df08bd09-c1e4-48b1-815c-cb9ad807607d","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:53.340324224Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"region":"df08bd09-c1e4-48b1-815c-cb9ad807607d","component":"worker"}
{"msg":"region df08bd09-c1e4-48b1-815c-cb9ad807607d state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:53.340368765Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:53.340565002Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-df08bd09-c1e4-48b1-815c-cb9ad807607d","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:53.342296228Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-df08bd09-c1e4-48b1-815c-cb9ad807607d","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:53.345756321Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:53.347134817Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"uri":"/crucible/0/regions","method":"POST","req_id":"07bfaca3-1194-426f-9361-8c6bcfce9ac1","remote_addr":"[fd00:1122:3344:101::a]:61183","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":65,"response_code":200}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:53.357653541Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-df08bd09-c1e4-48b1-815c-cb9ad807607d","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:53.362246901Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/df08bd09-c1e4-48b1-815c-cb9ad807607d","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:53.362657674Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:53.362735247Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:53.362799939Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:53.362862891Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:53.369334713Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:58:53.382460202Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:59:19.23590529Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"uri":"/crucible/0/regions/df08bd09-c1e4-48b1-815c-cb9ad807607d","method":"GET","req_id":"6fac9676-e07d-45b0-ae8c-fbd495829699","remote_addr":"[fd00:1122:3344:101::a]:61183","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":53,"response_code":200}
{"msg":"path is \"/data/regions/df08bd09-c1e4-48b1-815c-cb9ad807607d\"","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:59:19.239920532Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"datafile"}
{"msg":"dataset is oxp_7260b4b3-a573-4f9c-9e86-8952e72fc71e/crucible/regions/df08bd09-c1e4-48b1-815c-cb9ad807607d","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:59:19.251014645Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"datafile"}
{"msg":"region df08bd09-c1e4-48b1-815c-cb9ad807607d state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:59:19.25850141Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:59:19.259218233Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"uri":"/crucible/0/regions/df08bd09-c1e4-48b1-815c-cb9ad807607d","method":"DELETE","req_id":"c117279f-b40f-4156-a5c2-d0aa2f5e6b8f","remote_addr":"[fd00:1122:3344:101::a]:61183","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":19333,"response_code":204}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:59:19.259249874Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-df08bd09-c1e4-48b1-815c-cb9ad807607d (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:59:19.26033884Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:59:19.262222091Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"uri":"/crucible/0/regions/df08bd09-c1e4-48b1-815c-cb9ad807607d","method":"GET","req_id":"5baef605-5c9d-43ad-81c7-2007d51b32bc","remote_addr":"[fd00:1122:3344:101::a]:61183","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":41,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:59:19.2685909Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_7260b4b3-a573-4f9c-9e86-8952e72fc71e/crucible/regions/df08bd09-c1e4-48b1-815c-cb9ad807607d\"","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:59:19.276198139Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"region":"df08bd09-c1e4-48b1-815c-cb9ad807607d","component":"worker"}
{"msg":"region df08bd09-c1e4-48b1-815c-cb9ad807607d state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:59:19.329607286Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-28T02:59:19.399167471Z","hostname":"oxz_crucible_89e6f4e6-8ed9-4570-956c-d310ef94b51a","pid":24403,"uri":"/crucible/0/regions/df08bd09-c1e4-48b1-815c-cb9ad807607d","method":"GET","req_id":"1ea9a5b6-ceb0-4f8c-b4b8-da06ab2c8e86","remote_addr":"[fd00:1122:3344:101::a]:61183","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":44,"response_code":200}