[ Jul 23 10:13:48 Enabled. ] [ Jul 23 10:13:48 Rereading configuration. ] [ Jul 23 10:13:49 Rereading configuration. ] [ Jul 23 10:13:53 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_d44e746a-5ffe-4aa4-bc73-16769d833506/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"dataset: \"oxp_d44e746a-5ffe-4aa4-bc73-16769d833506/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:13:53.500599421Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893} {"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:13:53.505010882Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:13:53.505020972Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:13:53.530541785Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:13:53.633053988Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:48.501720929Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:35269"} {"msg":"region a3a914a0-c7ca-4bd8-9d96-78dd3d1384d4 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:48.529697562Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:48.534019401Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"uri":"/crucible/0/regions","method":"POST","req_id":"0c736607-4b7b-483b-8885-8b0ba3f945ca","remote_addr":"[fd00:1122:3344:101::a]:35269","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":22505,"response_code":"200"} {"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:48.536270497Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"component":"worker"} {"msg":"zfs set reservation of 1342177280 for oxp_d44e746a-5ffe-4aa4-bc73-16769d833506/crucible/regions/a3a914a0-c7ca-4bd8-9d96-78dd3d1384d4","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:48.545566878Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"component":"worker"} {"msg":"zfs set quota of 3221225472 for oxp_d44e746a-5ffe-4aa4-bc73-16769d833506/crucible/regions/a3a914a0-c7ca-4bd8-9d96-78dd3d1384d4","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:48.545598838Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"a3a914a0-c7ca-4bd8-9d96-78dd3d1384d4\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/a3a914a0-c7ca-4bd8-9d96-78dd3d1384d4\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:48.605019357Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"region":"a3a914a0-c7ca-4bd8-9d96-78dd3d1384d4","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"a3a914a0-c7ca-4bd8-9d96-78dd3d1384d4\" \"--data\" \"/data/regions/a3a914a0-c7ca-4bd8-9d96-78dd3d1384d4\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:48.605078068Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"region":"a3a914a0-c7ca-4bd8-9d96-78dd3d1384d4","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:48.854238837Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"uri":"/crucible/0/regions","method":"POST","req_id":"d2e4a865-ba38-4486-b619-b06a980091f1","remote_addr":"[fd00:1122:3344:101::a]:35269","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":68,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:48.894451702Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"region":"a3a914a0-c7ca-4bd8-9d96-78dd3d1384d4","component":"worker"} {"msg":"region a3a914a0-c7ca-4bd8-9d96-78dd3d1384d4 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:48.894490553Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:48.894590985Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-a3a914a0-c7ca-4bd8-9d96-78dd3d1384d4","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:48.894953752Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-a3a914a0-c7ca-4bd8-9d96-78dd3d1384d4","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:48.899991729Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:48.915478803Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a3a914a0-c7ca-4bd8-9d96-78dd3d1384d4","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:48.9201951Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a3a914a0-c7ca-4bd8-9d96-78dd3d1384d4","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:48.920922955Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:48.920981686Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:48.921023347Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:48.921059108Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:48.926209573Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:48.937055746Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:49.244963Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"uri":"/crucible/0/regions","method":"POST","req_id":"f4b7af05-6fe5-4847-b623-cda4fdc4c442","remote_addr":"[fd00:1122:3344:101::a]:35269","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":45,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:49.675284524Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"uri":"/crucible/0/regions/a3a914a0-c7ca-4bd8-9d96-78dd3d1384d4","method":"GET","req_id":"f2a5beea-babf-4a80-a56d-338d83e2fa8d","remote_addr":"[fd00:1122:3344:101::a]:35269","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":83,"response_code":"200"} {"msg":"path is \"/data/regions/a3a914a0-c7ca-4bd8-9d96-78dd3d1384d4\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:49.678222684Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"component":"datafile"} {"msg":"dataset is oxp_d44e746a-5ffe-4aa4-bc73-16769d833506/crucible/regions/a3a914a0-c7ca-4bd8-9d96-78dd3d1384d4","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:49.689249561Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"component":"datafile"} {"msg":"region a3a914a0-c7ca-4bd8-9d96-78dd3d1384d4 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:49.69943473Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:49.699590713Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:49.699648684Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"uri":"/crucible/0/regions/a3a914a0-c7ca-4bd8-9d96-78dd3d1384d4","method":"DELETE","req_id":"21cf9d29-8fb6-4d46-9421-c24be6ef562a","remote_addr":"[fd00:1122:3344:101::a]:35269","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":21458,"response_code":"204"} {"msg":"disabling downstairs instance: downstairs-a3a914a0-c7ca-4bd8-9d96-78dd3d1384d4 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:49.700551513Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:49.702316809Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"uri":"/crucible/0/regions/a3a914a0-c7ca-4bd8-9d96-78dd3d1384d4","method":"GET","req_id":"9c739ed0-77c4-42cd-87e0-bedc33b5e388","remote_addr":"[fd00:1122:3344:101::a]:35269","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":25,"response_code":"200"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:49.712526288Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_d44e746a-5ffe-4aa4-bc73-16769d833506/crucible/regions/a3a914a0-c7ca-4bd8-9d96-78dd3d1384d4\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:49.724596366Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"region":"a3a914a0-c7ca-4bd8-9d96-78dd3d1384d4","component":"worker"} {"msg":"region a3a914a0-c7ca-4bd8-9d96-78dd3d1384d4 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:49.787471745Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:49.810320609Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"uri":"/crucible/0/regions/a3a914a0-c7ca-4bd8-9d96-78dd3d1384d4","method":"GET","req_id":"cf85883e-a0ba-4f76-8dc5-f6b4c518ffa2","remote_addr":"[fd00:1122:3344:101::a]:35269","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":58,"response_code":"200"} {"msg":"region f5af0545-3c0a-47cb-ad64-cc0ee5757674 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:52.092045684Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:52.092192477Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:52.092209497Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"uri":"/crucible/0/regions","method":"POST","req_id":"bfae3609-acc5-48df-9248-003027c088f9","remote_addr":"[fd00:1122:3344:101::a]:35269","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":214,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_d44e746a-5ffe-4aa4-bc73-16769d833506/crucible/regions/f5af0545-3c0a-47cb-ad64-cc0ee5757674","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:52.102275934Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_d44e746a-5ffe-4aa4-bc73-16769d833506/crucible/regions/f5af0545-3c0a-47cb-ad64-cc0ee5757674","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:52.102306064Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"f5af0545-3c0a-47cb-ad64-cc0ee5757674\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/f5af0545-3c0a-47cb-ad64-cc0ee5757674\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:52.169741157Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"region":"f5af0545-3c0a-47cb-ad64-cc0ee5757674","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"f5af0545-3c0a-47cb-ad64-cc0ee5757674\" \"--data\" \"/data/regions/f5af0545-3c0a-47cb-ad64-cc0ee5757674\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:52.171771639Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"region":"f5af0545-3c0a-47cb-ad64-cc0ee5757674","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:52.2713335Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"region":"f5af0545-3c0a-47cb-ad64-cc0ee5757674","component":"worker"} {"msg":"region f5af0545-3c0a-47cb-ad64-cc0ee5757674 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:52.271385022Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:52.271475043Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-f5af0545-3c0a-47cb-ad64-cc0ee5757674","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:52.272735909Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f5af0545-3c0a-47cb-ad64-cc0ee5757674","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:52.277090209Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:52.284709205Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"uri":"/crucible/0/regions","method":"POST","req_id":"afed5387-51f9-445d-8077-318ed8f58c82","remote_addr":"[fd00:1122:3344:101::a]:35269","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":56,"response_code":"200"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:52.284733725Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f5af0545-3c0a-47cb-ad64-cc0ee5757674","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:52.288573144Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f5af0545-3c0a-47cb-ad64-cc0ee5757674","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:52.288902861Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:52.288949942Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:52.289002363Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:52.289042694Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:52.294899194Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T10:14:52.300810704Z","hostname":"oxz_crucible_2443119a-4e5e-40bb-a367-7ae3288adead","pid":22893,"component":"worker"}