[ Aug 28 23:27:23 Enabled. ]
[ Aug 28 23:27:23 Rereading configuration. ]
[ Aug 28 23:27:27 Rereading configuration. ]
[ Aug 28 23:27:29 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_cbb06c36-9ad5-4889-87cd-de8ad3eee08c/crucible -l [fd00:1122:3344:101::17]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_cbb06c36-9ad5-4889-87cd-de8ad3eee08c/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-28T23:27:29.296698428Z","hostname":"oxz_crucible_e179ca38-a415-471e-8800-bb2b5cf239ad","pid":23378}
{"msg":"listen IP: [fd00:1122:3344:101::17]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-08-28T23:27:29.297778778Z","hostname":"oxz_crucible_e179ca38-a415-471e-8800-bb2b5cf239ad","pid":23378}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-28T23:27:29.297789828Z","hostname":"oxz_crucible_e179ca38-a415-471e-8800-bb2b5cf239ad","pid":23378}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-28T23:27:29.314762447Z","hostname":"oxz_crucible_e179ca38-a415-471e-8800-bb2b5cf239ad","pid":23378,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-08-28T23:27:29.787726674Z","hostname":"oxz_crucible_e179ca38-a415-471e-8800-bb2b5cf239ad","pid":23378,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-28T23:28:01.472402778Z","hostname":"oxz_crucible_e179ca38-a415-471e-8800-bb2b5cf239ad","pid":23378,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:39289"}
{"msg":"region a765460c-a60c-4d81-bbbd-7f261c941ec9 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-28T23:28:01.476471003Z","hostname":"oxz_crucible_e179ca38-a415-471e-8800-bb2b5cf239ad","pid":23378,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-28T23:28:01.476902531Z","hostname":"oxz_crucible_e179ca38-a415-471e-8800-bb2b5cf239ad","pid":23378,"uri":"/crucible/0/regions","method":"POST","req_id":"d0a39545-b5fe-428d-b768-6bb9dac54f32","remote_addr":"[fd00:1122:3344:101::b]:39289","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":908,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-08-28T23:28:01.477477101Z","hostname":"oxz_crucible_e179ca38-a415-471e-8800-bb2b5cf239ad","pid":23378,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_cbb06c36-9ad5-4889-87cd-de8ad3eee08c/crucible/regions/a765460c-a60c-4d81-bbbd-7f261c941ec9","v":0,"name":"crucible-agent","level":30,"time":"2024-08-28T23:28:01.488041203Z","hostname":"oxz_crucible_e179ca38-a415-471e-8800-bb2b5cf239ad","pid":23378,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_cbb06c36-9ad5-4889-87cd-de8ad3eee08c/crucible/regions/a765460c-a60c-4d81-bbbd-7f261c941ec9","v":0,"name":"crucible-agent","level":30,"time":"2024-08-28T23:28:01.488077104Z","hostname":"oxz_crucible_e179ca38-a415-471e-8800-bb2b5cf239ad","pid":23378,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"a765460c-a60c-4d81-bbbd-7f261c941ec9\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/a765460c-a60c-4d81-bbbd-7f261c941ec9\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-28T23:28:01.544206477Z","hostname":"oxz_crucible_e179ca38-a415-471e-8800-bb2b5cf239ad","pid":23378,"region":"a765460c-a60c-4d81-bbbd-7f261c941ec9","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"a765460c-a60c-4d81-bbbd-7f261c941ec9\" \"--data\" \"/data/regions/a765460c-a60c-4d81-bbbd-7f261c941ec9\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-28T23:28:01.544268728Z","hostname":"oxz_crucible_e179ca38-a415-471e-8800-bb2b5cf239ad","pid":23378,"region":"a765460c-a60c-4d81-bbbd-7f261c941ec9","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-28T23:28:01.653833094Z","hostname":"oxz_crucible_e179ca38-a415-471e-8800-bb2b5cf239ad","pid":23378,"region":"a765460c-a60c-4d81-bbbd-7f261c941ec9","component":"worker"}
{"msg":"region a765460c-a60c-4d81-bbbd-7f261c941ec9 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-28T23:28:01.653890915Z","hostname":"oxz_crucible_e179ca38-a415-471e-8800-bb2b5cf239ad","pid":23378,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-28T23:28:01.653976867Z","hostname":"oxz_crucible_e179ca38-a415-471e-8800-bb2b5cf239ad","pid":23378,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-28T23:28:01.654102079Z","hostname":"oxz_crucible_e179ca38-a415-471e-8800-bb2b5cf239ad","pid":23378,"uri":"/crucible/0/regions","method":"POST","req_id":"ecc6a0b3-f54b-4e97-bc34-966cc7f9a6cf","remote_addr":"[fd00:1122:3344:101::b]:39289","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":76,"response_code":"200"}
{"msg":"creating missing downstairs instance downstairs-a765460c-a60c-4d81-bbbd-7f261c941ec9","v":0,"name":"crucible-agent","level":30,"time":"2024-08-28T23:28:01.654404045Z","hostname":"oxz_crucible_e179ca38-a415-471e-8800-bb2b5cf239ad","pid":23378,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-a765460c-a60c-4d81-bbbd-7f261c941ec9","v":0,"name":"crucible-agent","level":30,"time":"2024-08-28T23:28:01.657831327Z","hostname":"oxz_crucible_e179ca38-a415-471e-8800-bb2b5cf239ad","pid":23378,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-28T23:28:01.669124923Z","hostname":"oxz_crucible_e179ca38-a415-471e-8800-bb2b5cf239ad","pid":23378,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a765460c-a60c-4d81-bbbd-7f261c941ec9","v":0,"name":"crucible-agent","level":30,"time":"2024-08-28T23:28:01.672641397Z","hostname":"oxz_crucible_e179ca38-a415-471e-8800-bb2b5cf239ad","pid":23378,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a765460c-a60c-4d81-bbbd-7f261c941ec9","v":0,"name":"crucible-agent","level":30,"time":"2024-08-28T23:28:01.673519263Z","hostname":"oxz_crucible_e179ca38-a415-471e-8800-bb2b5cf239ad","pid":23378,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-28T23:28:01.673611304Z","hostname":"oxz_crucible_e179ca38-a415-471e-8800-bb2b5cf239ad","pid":23378,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-08-28T23:28:01.673648165Z","hostname":"oxz_crucible_e179ca38-a415-471e-8800-bb2b5cf239ad","pid":23378,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-28T23:28:01.673692496Z","hostname":"oxz_crucible_e179ca38-a415-471e-8800-bb2b5cf239ad","pid":23378,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-28T23:28:01.68049968Z","hostname":"oxz_crucible_e179ca38-a415-471e-8800-bb2b5cf239ad","pid":23378,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-28T23:28:01.686300546Z","hostname":"oxz_crucible_e179ca38-a415-471e-8800-bb2b5cf239ad","pid":23378,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-28T23:28:02.690239065Z","hostname":"oxz_crucible_e179ca38-a415-471e-8800-bb2b5cf239ad","pid":23378,"uri":"/crucible/0/regions/a765460c-a60c-4d81-bbbd-7f261c941ec9","method":"GET","req_id":"b8187123-44ba-4b03-ab28-e2117c0d52e0","remote_addr":"[fd00:1122:3344:101::b]:39289","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":91,"response_code":"200"}
{"msg":"path is \"/data/regions/a765460c-a60c-4d81-bbbd-7f261c941ec9\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-28T23:28:02.694379781Z","hostname":"oxz_crucible_e179ca38-a415-471e-8800-bb2b5cf239ad","pid":23378,"component":"datafile"}
{"msg":"dataset is oxp_cbb06c36-9ad5-4889-87cd-de8ad3eee08c/crucible/regions/a765460c-a60c-4d81-bbbd-7f261c941ec9","v":0,"name":"crucible-agent","level":30,"time":"2024-08-28T23:28:02.70589376Z","hostname":"oxz_crucible_e179ca38-a415-471e-8800-bb2b5cf239ad","pid":23378,"component":"datafile"}
{"msg":"region a765460c-a60c-4d81-bbbd-7f261c941ec9 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-08-28T23:28:02.716451533Z","hostname":"oxz_crucible_e179ca38-a415-471e-8800-bb2b5cf239ad","pid":23378,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-28T23:28:02.716618966Z","hostname":"oxz_crucible_e179ca38-a415-471e-8800-bb2b5cf239ad","pid":23378,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-28T23:28:02.716710047Z","hostname":"oxz_crucible_e179ca38-a415-471e-8800-bb2b5cf239ad","pid":23378,"uri":"/crucible/0/regions/a765460c-a60c-4d81-bbbd-7f261c941ec9","method":"DELETE","req_id":"40342148-53b0-466f-b089-198a55960589","remote_addr":"[fd00:1122:3344:101::b]:39289","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":22351,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-a765460c-a60c-4d81-bbbd-7f261c941ec9 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-08-28T23:28:02.717814247Z","hostname":"oxz_crucible_e179ca38-a415-471e-8800-bb2b5cf239ad","pid":23378,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-28T23:28:02.721164389Z","hostname":"oxz_crucible_e179ca38-a415-471e-8800-bb2b5cf239ad","pid":23378,"uri":"/crucible/0/regions/a765460c-a60c-4d81-bbbd-7f261c941ec9","method":"GET","req_id":"daceb6a0-fc01-4c8c-94ca-8e318f5285f4","remote_addr":"[fd00:1122:3344:101::b]:39289","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":619,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-28T23:28:02.722071915Z","hostname":"oxz_crucible_e179ca38-a415-471e-8800-bb2b5cf239ad","pid":23378,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_cbb06c36-9ad5-4889-87cd-de8ad3eee08c/crucible/regions/a765460c-a60c-4d81-bbbd-7f261c941ec9\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-28T23:28:02.733526174Z","hostname":"oxz_crucible_e179ca38-a415-471e-8800-bb2b5cf239ad","pid":23378,"region":"a765460c-a60c-4d81-bbbd-7f261c941ec9","component":"worker"}
{"msg":"region a765460c-a60c-4d81-bbbd-7f261c941ec9 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-28T23:28:02.7892862Z","hostname":"oxz_crucible_e179ca38-a415-471e-8800-bb2b5cf239ad","pid":23378,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-28T23:28:02.807728126Z","hostname":"oxz_crucible_e179ca38-a415-471e-8800-bb2b5cf239ad","pid":23378,"uri":"/crucible/0/regions/a765460c-a60c-4d81-bbbd-7f261c941ec9","method":"GET","req_id":"bd804bb6-76d1-4b71-a48b-f95eac3eb570","remote_addr":"[fd00:1122:3344:101::b]:39289","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":86,"response_code":"200"}