[ Nov 25 19:31:02 Enabled. ]
[ Nov 25 19:31:02 Rereading configuration. ]
[ Nov 25 19:31:05 Rereading configuration. ]
[ Nov 25 19:31:09 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_56aea45f-c451-4174-9b8f-66145c2696e8/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_56aea45f-c451-4174-9b8f-66145c2696e8/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:09.895834048Z","hostname":"oxz_crucible_d6f634f5-3528-4e41-966f-f47648523f6a","pid":28479}
{"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:09.899068921Z","hostname":"oxz_crucible_d6f634f5-3528-4e41-966f-f47648523f6a","pid":28479}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:09.899079781Z","hostname":"oxz_crucible_d6f634f5-3528-4e41-966f-f47648523f6a","pid":28479}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:09.914969333Z","hostname":"oxz_crucible_d6f634f5-3528-4e41-966f-f47648523f6a","pid":28479,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:31:09.990034666Z","hostname":"oxz_crucible_d6f634f5-3528-4e41-966f-f47648523f6a","pid":28479,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:35.476855182Z","hostname":"oxz_crucible_d6f634f5-3528-4e41-966f-f47648523f6a","pid":28479,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:44408"}
{"msg":"region 917e4ee4-3c72-4c2e-853d-94bc9192cf27 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:35.525233721Z","hostname":"oxz_crucible_d6f634f5-3528-4e41-966f-f47648523f6a","pid":28479,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:35.53028848Z","hostname":"oxz_crucible_d6f634f5-3528-4e41-966f-f47648523f6a","pid":28479,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:35.535801178Z","hostname":"oxz_crucible_d6f634f5-3528-4e41-966f-f47648523f6a","pid":28479,"uri":"/crucible/0/regions","method":"POST","req_id":"74f8d6d6-8cc5-48b0-9425-8c27dfa69256","remote_addr":"[fd00:1122:3344:101::a]:44408","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":17616,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_56aea45f-c451-4174-9b8f-66145c2696e8/crucible/regions/917e4ee4-3c72-4c2e-853d-94bc9192cf27","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:35.545331795Z","hostname":"oxz_crucible_d6f634f5-3528-4e41-966f-f47648523f6a","pid":28479,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_56aea45f-c451-4174-9b8f-66145c2696e8/crucible/regions/917e4ee4-3c72-4c2e-853d-94bc9192cf27","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:35.545357286Z","hostname":"oxz_crucible_d6f634f5-3528-4e41-966f-f47648523f6a","pid":28479,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"917e4ee4-3c72-4c2e-853d-94bc9192cf27\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/917e4ee4-3c72-4c2e-853d-94bc9192cf27\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:35.600943956Z","hostname":"oxz_crucible_d6f634f5-3528-4e41-966f-f47648523f6a","pid":28479,"region":"917e4ee4-3c72-4c2e-853d-94bc9192cf27","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"917e4ee4-3c72-4c2e-853d-94bc9192cf27\" \"--data\" \"/data/regions/917e4ee4-3c72-4c2e-853d-94bc9192cf27\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:35.603425855Z","hostname":"oxz_crucible_d6f634f5-3528-4e41-966f-f47648523f6a","pid":28479,"region":"917e4ee4-3c72-4c2e-853d-94bc9192cf27","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:35.755366216Z","hostname":"oxz_crucible_d6f634f5-3528-4e41-966f-f47648523f6a","pid":28479,"uri":"/crucible/0/regions","method":"POST","req_id":"50a077dc-130b-4fdf-8b23-3e4764c9efa2","remote_addr":"[fd00:1122:3344:101::a]:44408","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":81,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:35.916669201Z","hostname":"oxz_crucible_d6f634f5-3528-4e41-966f-f47648523f6a","pid":28479,"region":"917e4ee4-3c72-4c2e-853d-94bc9192cf27","component":"worker"}
{"msg":"region 917e4ee4-3c72-4c2e-853d-94bc9192cf27 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:35.916727702Z","hostname":"oxz_crucible_d6f634f5-3528-4e41-966f-f47648523f6a","pid":28479,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:35.916832894Z","hostname":"oxz_crucible_d6f634f5-3528-4e41-966f-f47648523f6a","pid":28479,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-917e4ee4-3c72-4c2e-853d-94bc9192cf27","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:35.917270703Z","hostname":"oxz_crucible_d6f634f5-3528-4e41-966f-f47648523f6a","pid":28479,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-917e4ee4-3c72-4c2e-853d-94bc9192cf27","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:35.924925383Z","hostname":"oxz_crucible_d6f634f5-3528-4e41-966f-f47648523f6a","pid":28479,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:35.938070411Z","hostname":"oxz_crucible_d6f634f5-3528-4e41-966f-f47648523f6a","pid":28479,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-917e4ee4-3c72-4c2e-853d-94bc9192cf27","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:35.941189752Z","hostname":"oxz_crucible_d6f634f5-3528-4e41-966f-f47648523f6a","pid":28479,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/917e4ee4-3c72-4c2e-853d-94bc9192cf27","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:35.941759733Z","hostname":"oxz_crucible_d6f634f5-3528-4e41-966f-f47648523f6a","pid":28479,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:35.941817054Z","hostname":"oxz_crucible_d6f634f5-3528-4e41-966f-f47648523f6a","pid":28479,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:35.941858115Z","hostname":"oxz_crucible_d6f634f5-3528-4e41-966f-f47648523f6a","pid":28479,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:35.941896865Z","hostname":"oxz_crucible_d6f634f5-3528-4e41-966f-f47648523f6a","pid":28479,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:35.945382484Z","hostname":"oxz_crucible_d6f634f5-3528-4e41-966f-f47648523f6a","pid":28479,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:35.95181898Z","hostname":"oxz_crucible_d6f634f5-3528-4e41-966f-f47648523f6a","pid":28479,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:42:36.150434607Z","hostname":"oxz_crucible_d6f634f5-3528-4e41-966f-f47648523f6a","pid":28479,"uri":"/crucible/0/regions","method":"POST","req_id":"6738ba7b-421b-4941-8e55-598b9dd207bf","remote_addr":"[fd00:1122:3344:101::a]:44408","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":674,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:43:01.774191111Z","hostname":"oxz_crucible_d6f634f5-3528-4e41-966f-f47648523f6a","pid":28479,"uri":"/crucible/0/regions/917e4ee4-3c72-4c2e-853d-94bc9192cf27","method":"GET","req_id":"5cefaa54-ff92-4bd6-9403-68a32f2d8979","remote_addr":"[fd00:1122:3344:101::a]:44408","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":79,"response_code":"200"}
{"msg":"path is \"/data/regions/917e4ee4-3c72-4c2e-853d-94bc9192cf27\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:43:01.777973375Z","hostname":"oxz_crucible_d6f634f5-3528-4e41-966f-f47648523f6a","pid":28479,"component":"datafile"}
{"msg":"dataset is oxp_56aea45f-c451-4174-9b8f-66145c2696e8/crucible/regions/917e4ee4-3c72-4c2e-853d-94bc9192cf27","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:43:01.789098644Z","hostname":"oxz_crucible_d6f634f5-3528-4e41-966f-f47648523f6a","pid":28479,"component":"datafile"}
{"msg":"region 917e4ee4-3c72-4c2e-853d-94bc9192cf27 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:43:01.799468927Z","hostname":"oxz_crucible_d6f634f5-3528-4e41-966f-f47648523f6a","pid":28479,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:43:01.79964177Z","hostname":"oxz_crucible_d6f634f5-3528-4e41-966f-f47648523f6a","pid":28479,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:43:01.799708412Z","hostname":"oxz_crucible_d6f634f5-3528-4e41-966f-f47648523f6a","pid":28479,"uri":"/crucible/0/regions/917e4ee4-3c72-4c2e-853d-94bc9192cf27","method":"DELETE","req_id":"da9d159d-69f2-4acd-b440-da7ccf053fa6","remote_addr":"[fd00:1122:3344:101::a]:44408","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":21762,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-917e4ee4-3c72-4c2e-853d-94bc9192cf27 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:43:01.803407184Z","hostname":"oxz_crucible_d6f634f5-3528-4e41-966f-f47648523f6a","pid":28479,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:43:01.805307951Z","hostname":"oxz_crucible_d6f634f5-3528-4e41-966f-f47648523f6a","pid":28479,"uri":"/crucible/0/regions/917e4ee4-3c72-4c2e-853d-94bc9192cf27","method":"GET","req_id":"c444b639-5ca3-4ce7-8930-5e5cb8d05e37","remote_addr":"[fd00:1122:3344:101::a]:44408","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":779,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:43:01.808858251Z","hostname":"oxz_crucible_d6f634f5-3528-4e41-966f-f47648523f6a","pid":28479,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_56aea45f-c451-4174-9b8f-66145c2696e8/crucible/regions/917e4ee4-3c72-4c2e-853d-94bc9192cf27\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:43:01.82053009Z","hostname":"oxz_crucible_d6f634f5-3528-4e41-966f-f47648523f6a","pid":28479,"region":"917e4ee4-3c72-4c2e-853d-94bc9192cf27","component":"worker"}
{"msg":"region 917e4ee4-3c72-4c2e-853d-94bc9192cf27 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:43:01.8903539Z","hostname":"oxz_crucible_d6f634f5-3528-4e41-966f-f47648523f6a","pid":28479,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-25T19:43:01.923552351Z","hostname":"oxz_crucible_d6f634f5-3528-4e41-966f-f47648523f6a","pid":28479,"uri":"/crucible/0/regions/917e4ee4-3c72-4c2e-853d-94bc9192cf27","method":"GET","req_id":"24d5a073-5cfc-4de6-813a-06479bf155fa","remote_addr":"[fd00:1122:3344:101::a]:44408","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":82,"response_code":"200"}