[ Mar 26 00:01:33 Enabled. ]
[ Mar 26 00:01:33 Rereading configuration. ]
[ Mar 26 00:01:35 Rereading configuration. ]
[ Mar 26 00:01:38 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_f1c64d12-be3e-43ca-b969-46fee4cf27d9/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_f1c64d12-be3e-43ca-b969-46fee4cf27d9/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:38.854895958Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810}
{"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:38.858300728Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:38.858309818Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:38.878411743Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:38.968003909Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:54.382875502Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:37930"}
{"msg":"region b7949d17-adb4-4697-b396-a1c62281cd1f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:54.389762563Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:54.390637271Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"uri":"/crucible/0/regions","method":"POST","req_id":"633a9222-4017-4f20-843f-d4293e2b38c9","remote_addr":"[fd00:1122:3344:101::b]:37930","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":3589,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:54.390653861Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_f1c64d12-be3e-43ca-b969-46fee4cf27d9/crucible/regions/b7949d17-adb4-4697-b396-a1c62281cd1f","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:54.399717352Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_f1c64d12-be3e-43ca-b969-46fee4cf27d9/crucible/regions/b7949d17-adb4-4697-b396-a1c62281cd1f","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:54.399784954Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"b7949d17-adb4-4697-b396-a1c62281cd1f\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/b7949d17-adb4-4697-b396-a1c62281cd1f\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:54.480634959Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"region":"b7949d17-adb4-4697-b396-a1c62281cd1f","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"b7949d17-adb4-4697-b396-a1c62281cd1f\" \"--data\" \"/data/regions/b7949d17-adb4-4697-b396-a1c62281cd1f\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:54.480739892Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"region":"b7949d17-adb4-4697-b396-a1c62281cd1f","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:54.62803249Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"region":"b7949d17-adb4-4697-b396-a1c62281cd1f","component":"worker"}
{"msg":"region b7949d17-adb4-4697-b396-a1c62281cd1f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:54.628118213Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:54.62834039Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-b7949d17-adb4-4697-b396-a1c62281cd1f","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:54.628795804Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b7949d17-adb4-4697-b396-a1c62281cd1f","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:54.633101382Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:54.642599058Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b7949d17-adb4-4697-b396-a1c62281cd1f","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:54.647059061Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b7949d17-adb4-4697-b396-a1c62281cd1f","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:54.648108714Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:54.648194677Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:54.648251499Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:54.648300801Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:54.654569482Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:54.660363238Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:01:54.712855133Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"uri":"/crucible/0/regions","method":"POST","req_id":"c7924475-5268-44cf-80c1-b995ef715481","remote_addr":"[fd00:1122:3344:101::b]:37930","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":63,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:58.76466587Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:56959"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:58.767519011Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"uri":"/crucible/0/regions/b7949d17-adb4-4697-b396-a1c62281cd1f","method":"GET","req_id":"2c1f083d-232b-4a97-a83e-5f8bad2e8175","remote_addr":"[fd00:1122:3344:101::b]:56959","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":2817,"response_code":"200"}
{"msg":"path is \"/data/regions/b7949d17-adb4-4697-b396-a1c62281cd1f\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:58.772924075Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"datafile"}
{"msg":"dataset is oxp_f1c64d12-be3e-43ca-b969-46fee4cf27d9/crucible/regions/b7949d17-adb4-4697-b396-a1c62281cd1f","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:58.781569702Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"datafile"}
{"msg":"snapshot is oxp_f1c64d12-be3e-43ca-b969-46fee4cf27d9/crucible/regions/b7949d17-adb4-4697-b396-a1c62281cd1f@bdbcd4b1-312b-4e62-a120-390bab9f8ce0","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:58.789531448Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_f1c64d12-be3e-43ca-b969-46fee4cf27d9/crucible/regions/b7949d17-adb4-4697-b396-a1c62281cd1f\", \"bdbcd4b1-312b-4e62-a120-390bab9f8ce0\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:58.789565769Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:58.80143225Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"uri":"/crucible/0/regions/b7949d17-adb4-4697-b396-a1c62281cd1f/snapshots/bdbcd4b1-312b-4e62-a120-390bab9f8ce0","method":"GET","req_id":"828da5db-f023-4e61-9880-10bcca00e4db","remote_addr":"[fd00:1122:3344:101::b]:56959","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":30933,"response_code":"200"}
{"msg":"path is \"/data/regions/b7949d17-adb4-4697-b396-a1c62281cd1f\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:58.804749966Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"datafile"}
{"msg":"dataset is oxp_f1c64d12-be3e-43ca-b969-46fee4cf27d9/crucible/regions/b7949d17-adb4-4697-b396-a1c62281cd1f","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:58.814065075Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"datafile"}
{"msg":"snapshot is oxp_f1c64d12-be3e-43ca-b969-46fee4cf27d9/crucible/regions/b7949d17-adb4-4697-b396-a1c62281cd1f@bdbcd4b1-312b-4e62-a120-390bab9f8ce0","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:58.822871338Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_f1c64d12-be3e-43ca-b969-46fee4cf27d9/crucible/regions/b7949d17-adb4-4697-b396-a1c62281cd1f\", \"bdbcd4b1-312b-4e62-a120-390bab9f8ce0\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:58.822905509Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot b7949d17-adb4-4697-b396-a1c62281cd1f-bdbcd4b1-312b-4e62-a120-390bab9f8ce0 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:58.831755013Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"datafile"}
{"msg":"applying SMF actions for region b7949d17-adb4-4697-b396-a1c62281cd1f running snapshot bdbcd4b1-312b-4e62-a120-390bab9f8ce0 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:58.832383873Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:58.832409854Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"uri":"/crucible/0/regions/b7949d17-adb4-4697-b396-a1c62281cd1f/snapshots/bdbcd4b1-312b-4e62-a120-390bab9f8ce0/run","method":"POST","req_id":"15392748-cedf-4e1a-b847-c67197c7f90a","remote_addr":"[fd00:1122:3344:101::b]:56959","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":27691,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-b7949d17-adb4-4697-b396-a1c62281cd1f-bdbcd4b1-312b-4e62-a120-390bab9f8ce0","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:58.845745692Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-b7949d17-adb4-4697-b396-a1c62281cd1f-bdbcd4b1-312b-4e62-a120-390bab9f8ce0","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:58.850137733Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-b7949d17-adb4-4697-b396-a1c62281cd1f-bdbcd4b1-312b-4e62-a120-390bab9f8ce0","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:58.862755338Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b7949d17-adb4-4697-b396-a1c62281cd1f/.zfs/snapshot/bdbcd4b1-312b-4e62-a120-390bab9f8ce0","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:58.863046067Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:58.863097569Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:58.863139621Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:58.863178322Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:58.863225713Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:58.866772697Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:58.871596032Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"worker"}
{"msg":"region b7949d17-adb4-4697-b396-a1c62281cd1f running snapshot bdbcd4b1-312b-4e62-a120-390bab9f8ce0 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:58.871610572Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"datafile"}
{"msg":"path is \"/data/regions/b7949d17-adb4-4697-b396-a1c62281cd1f\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:58.982952726Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"datafile"}
{"msg":"dataset is oxp_f1c64d12-be3e-43ca-b969-46fee4cf27d9/crucible/regions/b7949d17-adb4-4697-b396-a1c62281cd1f","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:58.991383177Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"datafile"}
{"msg":"snapshot is oxp_f1c64d12-be3e-43ca-b969-46fee4cf27d9/crucible/regions/b7949d17-adb4-4697-b396-a1c62281cd1f@bdbcd4b1-312b-4e62-a120-390bab9f8ce0","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:58.999861729Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_f1c64d12-be3e-43ca-b969-46fee4cf27d9/crucible/regions/b7949d17-adb4-4697-b396-a1c62281cd1f\", \"bdbcd4b1-312b-4e62-a120-390bab9f8ce0\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:58.99990317Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-26T00:11:59.009088545Z","hostname":"oxz_crucible_c200782b-a2f5-4059-a1f5-3bb5992d6aa3","pid":26810,"uri":"/crucible/0/regions/b7949d17-adb4-4697-b396-a1c62281cd1f/snapshots/bdbcd4b1-312b-4e62-a120-390bab9f8ce0/run","method":"POST","req_id":"5b6778e9-0d4b-41a8-b4f0-dc5e9fe01a32","remote_addr":"[fd00:1122:3344:101::b]:56959","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":26770,"response_code":"200"}