[ Sep 27 04:18:19 Enabled. ]
[ Sep 27 04:18:19 Rereading configuration. ]
[ Sep 27 04:18:20 Rereading configuration. ]
[ Sep 27 04:18:23 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_48604733-34f0-4c7d-a800-bc2d35a9bd55/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_48604733-34f0-4c7d-a800-bc2d35a9bd55/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-27T04:18:23.787344153Z","hostname":"oxz_crucible_7d9c6025-fcf0-4bb5-a9f8-27441d950458","pid":23589}
{"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-09-27T04:18:23.790551082Z","hostname":"oxz_crucible_7d9c6025-fcf0-4bb5-a9f8-27441d950458","pid":23589}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-27T04:18:23.790567942Z","hostname":"oxz_crucible_7d9c6025-fcf0-4bb5-a9f8-27441d950458","pid":23589}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-27T04:18:23.810572119Z","hostname":"oxz_crucible_7d9c6025-fcf0-4bb5-a9f8-27441d950458","pid":23589,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-09-27T04:18:24.007122613Z","hostname":"oxz_crucible_7d9c6025-fcf0-4bb5-a9f8-27441d950458","pid":23589,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-09-27T04:24:24.3373509Z","hostname":"oxz_crucible_7d9c6025-fcf0-4bb5-a9f8-27441d950458","pid":23589,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:48415"}
{"msg":"region 635f694f-c42a-4b67-9fba-2b4e16844596 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-09-27T04:24:24.382329455Z","hostname":"oxz_crucible_7d9c6025-fcf0-4bb5-a9f8-27441d950458","pid":23589,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-27T04:24:24.387268235Z","hostname":"oxz_crucible_7d9c6025-fcf0-4bb5-a9f8-27441d950458","pid":23589,"uri":"/crucible/0/regions","method":"POST","req_id":"3bda1e26-825a-45b7-bf80-7a9c07c5a53e","remote_addr":"[fd00:1122:3344:101::a]:48415","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":30747,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-09-27T04:24:24.392173335Z","hostname":"oxz_crucible_7d9c6025-fcf0-4bb5-a9f8-27441d950458","pid":23589,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_48604733-34f0-4c7d-a800-bc2d35a9bd55/crucible/regions/635f694f-c42a-4b67-9fba-2b4e16844596","v":0,"name":"crucible-agent","level":30,"time":"2024-09-27T04:24:24.401809119Z","hostname":"oxz_crucible_7d9c6025-fcf0-4bb5-a9f8-27441d950458","pid":23589,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_48604733-34f0-4c7d-a800-bc2d35a9bd55/crucible/regions/635f694f-c42a-4b67-9fba-2b4e16844596","v":0,"name":"crucible-agent","level":30,"time":"2024-09-27T04:24:24.40183792Z","hostname":"oxz_crucible_7d9c6025-fcf0-4bb5-a9f8-27441d950458","pid":23589,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"635f694f-c42a-4b67-9fba-2b4e16844596\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/635f694f-c42a-4b67-9fba-2b4e16844596\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-27T04:24:24.453083077Z","hostname":"oxz_crucible_7d9c6025-fcf0-4bb5-a9f8-27441d950458","pid":23589,"region":"635f694f-c42a-4b67-9fba-2b4e16844596","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"635f694f-c42a-4b67-9fba-2b4e16844596\" \"--data\" \"/data/regions/635f694f-c42a-4b67-9fba-2b4e16844596\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-27T04:24:24.457615797Z","hostname":"oxz_crucible_7d9c6025-fcf0-4bb5-a9f8-27441d950458","pid":23589,"region":"635f694f-c42a-4b67-9fba-2b4e16844596","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-27T04:24:24.554144737Z","hostname":"oxz_crucible_7d9c6025-fcf0-4bb5-a9f8-27441d950458","pid":23589,"uri":"/crucible/0/regions","method":"POST","req_id":"bb6b684d-00fd-4aeb-a51e-692056142f3f","remote_addr":"[fd00:1122:3344:101::a]:48415","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":612,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-09-27T04:24:24.744557121Z","hostname":"oxz_crucible_7d9c6025-fcf0-4bb5-a9f8-27441d950458","pid":23589,"region":"635f694f-c42a-4b67-9fba-2b4e16844596","component":"worker"}
{"msg":"region 635f694f-c42a-4b67-9fba-2b4e16844596 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-09-27T04:24:24.744604582Z","hostname":"oxz_crucible_7d9c6025-fcf0-4bb5-a9f8-27441d950458","pid":23589,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-27T04:24:24.744709835Z","hostname":"oxz_crucible_7d9c6025-fcf0-4bb5-a9f8-27441d950458","pid":23589,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-635f694f-c42a-4b67-9fba-2b4e16844596","v":0,"name":"crucible-agent","level":30,"time":"2024-09-27T04:24:24.747403081Z","hostname":"oxz_crucible_7d9c6025-fcf0-4bb5-a9f8-27441d950458","pid":23589,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-635f694f-c42a-4b67-9fba-2b4e16844596","v":0,"name":"crucible-agent","level":30,"time":"2024-09-27T04:24:24.753003677Z","hostname":"oxz_crucible_7d9c6025-fcf0-4bb5-a9f8-27441d950458","pid":23589,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-27T04:24:24.768527584Z","hostname":"oxz_crucible_7d9c6025-fcf0-4bb5-a9f8-27441d950458","pid":23589,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-635f694f-c42a-4b67-9fba-2b4e16844596","v":0,"name":"crucible-agent","level":30,"time":"2024-09-27T04:24:24.771911747Z","hostname":"oxz_crucible_7d9c6025-fcf0-4bb5-a9f8-27441d950458","pid":23589,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/635f694f-c42a-4b67-9fba-2b4e16844596","v":0,"name":"crucible-agent","level":30,"time":"2024-09-27T04:24:24.782243818Z","hostname":"oxz_crucible_7d9c6025-fcf0-4bb5-a9f8-27441d950458","pid":23589,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-27T04:24:24.784853162Z","hostname":"oxz_crucible_7d9c6025-fcf0-4bb5-a9f8-27441d950458","pid":23589,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-09-27T04:24:24.784903923Z","hostname":"oxz_crucible_7d9c6025-fcf0-4bb5-a9f8-27441d950458","pid":23589,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-27T04:24:24.784941424Z","hostname":"oxz_crucible_7d9c6025-fcf0-4bb5-a9f8-27441d950458","pid":23589,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-27T04:24:24.788993903Z","hostname":"oxz_crucible_7d9c6025-fcf0-4bb5-a9f8-27441d950458","pid":23589,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-27T04:24:24.796050014Z","hostname":"oxz_crucible_7d9c6025-fcf0-4bb5-a9f8-27441d950458","pid":23589,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-27T04:24:25.04691767Z","hostname":"oxz_crucible_7d9c6025-fcf0-4bb5-a9f8-27441d950458","pid":23589,"uri":"/crucible/0/regions","method":"POST","req_id":"a89ecbee-bdf6-40e1-9d21-820ed1b2b1a4","remote_addr":"[fd00:1122:3344:101::a]:48415","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":60,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-27T04:24:52.297801912Z","hostname":"oxz_crucible_7d9c6025-fcf0-4bb5-a9f8-27441d950458","pid":23589,"uri":"/crucible/0/regions/635f694f-c42a-4b67-9fba-2b4e16844596","method":"GET","req_id":"accb89d4-2805-4216-a314-c4c010d95051","remote_addr":"[fd00:1122:3344:101::a]:48415","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":78,"response_code":"200"}
{"msg":"path is \"/data/regions/635f694f-c42a-4b67-9fba-2b4e16844596\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-27T04:24:52.300927578Z","hostname":"oxz_crucible_7d9c6025-fcf0-4bb5-a9f8-27441d950458","pid":23589,"component":"datafile"}
{"msg":"dataset is oxp_48604733-34f0-4c7d-a800-bc2d35a9bd55/crucible/regions/635f694f-c42a-4b67-9fba-2b4e16844596","v":0,"name":"crucible-agent","level":30,"time":"2024-09-27T04:24:52.312639523Z","hostname":"oxz_crucible_7d9c6025-fcf0-4bb5-a9f8-27441d950458","pid":23589,"component":"datafile"}
{"msg":"region 635f694f-c42a-4b67-9fba-2b4e16844596 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-09-27T04:24:52.323331604Z","hostname":"oxz_crucible_7d9c6025-fcf0-4bb5-a9f8-27441d950458","pid":23589,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-27T04:24:52.323515048Z","hostname":"oxz_crucible_7d9c6025-fcf0-4bb5-a9f8-27441d950458","pid":23589,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-27T04:24:52.32358532Z","hostname":"oxz_crucible_7d9c6025-fcf0-4bb5-a9f8-27441d950458","pid":23589,"uri":"/crucible/0/regions/635f694f-c42a-4b67-9fba-2b4e16844596","method":"DELETE","req_id":"38b4cdd7-4582-4e26-8e5e-aca4ee3a9242","remote_addr":"[fd00:1122:3344:101::a]:48415","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":23039,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-635f694f-c42a-4b67-9fba-2b4e16844596 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-09-27T04:24:52.324616185Z","hostname":"oxz_crucible_7d9c6025-fcf0-4bb5-a9f8-27441d950458","pid":23589,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-27T04:24:52.326904521Z","hostname":"oxz_crucible_7d9c6025-fcf0-4bb5-a9f8-27441d950458","pid":23589,"uri":"/crucible/0/regions/635f694f-c42a-4b67-9fba-2b4e16844596","method":"GET","req_id":"3a5fddac-4273-4188-8214-e8e64c8f5040","remote_addr":"[fd00:1122:3344:101::a]:48415","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":24,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-27T04:24:52.329600666Z","hostname":"oxz_crucible_7d9c6025-fcf0-4bb5-a9f8-27441d950458","pid":23589,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_48604733-34f0-4c7d-a800-bc2d35a9bd55/crucible/regions/635f694f-c42a-4b67-9fba-2b4e16844596\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-27T04:24:52.340511042Z","hostname":"oxz_crucible_7d9c6025-fcf0-4bb5-a9f8-27441d950458","pid":23589,"region":"635f694f-c42a-4b67-9fba-2b4e16844596","component":"worker"}
{"msg":"region 635f694f-c42a-4b67-9fba-2b4e16844596 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-27T04:24:52.399864406Z","hostname":"oxz_crucible_7d9c6025-fcf0-4bb5-a9f8-27441d950458","pid":23589,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-27T04:24:52.419357871Z","hostname":"oxz_crucible_7d9c6025-fcf0-4bb5-a9f8-27441d950458","pid":23589,"uri":"/crucible/0/regions/635f694f-c42a-4b67-9fba-2b4e16844596","method":"GET","req_id":"5f724c68-146f-465d-ac7a-39463fd5c6e3","remote_addr":"[fd00:1122:3344:101::a]:48415","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":631,"response_code":"200"}