[ Dec 20 21:21:59 Enabled. ] [ Dec 20 21:21:59 Rereading configuration. ] [ Dec 20 21:22:01 Rereading configuration. ] [ Dec 20 21:22:05 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_b35cb2aa-7e4b-4820-82a1-1376e0cb1176/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"dataset: \"oxp_b35cb2aa-7e4b-4820-82a1-1376e0cb1176/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:22:05.604343377Z","hostname":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","pid":23730} {"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:22:05.608319536Z","hostname":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","pid":23730} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:22:05.608332097Z","hostname":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","pid":23730} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:22:05.623863075Z","hostname":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","pid":23730,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:22:05.691456351Z","hostname":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","pid":23730,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:23:08.985972679Z","hostname":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","pid":23730,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:41817"} {"msg":"region f70433db-9260-4cb7-8655-b2e0ab3333e8 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:23:09.005024236Z","hostname":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","pid":23730,"component":"datafile"} {"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:23:09.007608414Z","hostname":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","pid":23730,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:23:09.012228948Z","hostname":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","pid":23730,"uri":"/crucible/0/regions","method":"POST","req_id":"0291cf45-0204-4d02-850c-8366cf88d5b5","remote_addr":"[fd00:1122:3344:101::a]:41817","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":9335,"response_code":"200"} {"msg":"zfs set reservation of 1342177280 for oxp_b35cb2aa-7e4b-4820-82a1-1376e0cb1176/crucible/regions/f70433db-9260-4cb7-8655-b2e0ab3333e8","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:23:09.017978717Z","hostname":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","pid":23730,"component":"worker"} {"msg":"zfs set quota of 3221225472 for oxp_b35cb2aa-7e4b-4820-82a1-1376e0cb1176/crucible/regions/f70433db-9260-4cb7-8655-b2e0ab3333e8","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:23:09.017998558Z","hostname":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","pid":23730,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"f70433db-9260-4cb7-8655-b2e0ab3333e8\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/f70433db-9260-4cb7-8655-b2e0ab3333e8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:23:09.075006506Z","hostname":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","pid":23730,"region":"f70433db-9260-4cb7-8655-b2e0ab3333e8","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"f70433db-9260-4cb7-8655-b2e0ab3333e8\" \"--data\" \"/data/regions/f70433db-9260-4cb7-8655-b2e0ab3333e8\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:23:09.077208385Z","hostname":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","pid":23730,"region":"f70433db-9260-4cb7-8655-b2e0ab3333e8","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:23:09.195670971Z","hostname":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","pid":23730,"region":"f70433db-9260-4cb7-8655-b2e0ab3333e8","component":"worker"} {"msg":"region f70433db-9260-4cb7-8655-b2e0ab3333e8 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:23:09.195721713Z","hostname":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","pid":23730,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:23:09.195789274Z","hostname":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","pid":23730,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-f70433db-9260-4cb7-8655-b2e0ab3333e8","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:23:09.196162743Z","hostname":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","pid":23730,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f70433db-9260-4cb7-8655-b2e0ab3333e8","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:23:09.202509325Z","hostname":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","pid":23730,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:23:09.213076102Z","hostname":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","pid":23730,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f70433db-9260-4cb7-8655-b2e0ab3333e8","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:23:09.21700163Z","hostname":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","pid":23730,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f70433db-9260-4cb7-8655-b2e0ab3333e8","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:23:09.217718726Z","hostname":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","pid":23730,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:23:09.217785898Z","hostname":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","pid":23730,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:23:09.217830759Z","hostname":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","pid":23730,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:23:09.217870199Z","hostname":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","pid":23730,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:23:09.223879854Z","hostname":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","pid":23730,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:23:09.228353755Z","hostname":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","pid":23730,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:23:09.296268107Z","hostname":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","pid":23730,"uri":"/crucible/0/regions","method":"POST","req_id":"055e1402-281f-427f-a8a7-dbdbb013c699","remote_addr":"[fd00:1122:3344:101::a]:41817","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":62,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:23:09.667638085Z","hostname":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","pid":23730,"uri":"/crucible/0/regions/f70433db-9260-4cb7-8655-b2e0ab3333e8","method":"GET","req_id":"d1254e3c-adbe-4f60-8084-94a0b39fc823","remote_addr":"[fd00:1122:3344:101::a]:41817","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":71,"response_code":"200"} {"msg":"path is \"/data/regions/f70433db-9260-4cb7-8655-b2e0ab3333e8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:23:09.670316315Z","hostname":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","pid":23730,"component":"datafile"} {"msg":"dataset is oxp_b35cb2aa-7e4b-4820-82a1-1376e0cb1176/crucible/regions/f70433db-9260-4cb7-8655-b2e0ab3333e8","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:23:09.681478865Z","hostname":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","pid":23730,"component":"datafile"} {"msg":"region f70433db-9260-4cb7-8655-b2e0ab3333e8 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:23:09.692813419Z","hostname":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","pid":23730,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:23:09.692967593Z","hostname":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","pid":23730,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:23:09.693033084Z","hostname":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","pid":23730,"uri":"/crucible/0/regions/f70433db-9260-4cb7-8655-b2e0ab3333e8","method":"DELETE","req_id":"a0bc8a76-7d03-4039-bc4c-cc18431f69e2","remote_addr":"[fd00:1122:3344:101::a]:41817","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":22744,"response_code":"204"} {"msg":"disabling downstairs instance: downstairs-f70433db-9260-4cb7-8655-b2e0ab3333e8 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:23:09.696049312Z","hostname":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","pid":23730,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:23:09.696436651Z","hostname":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","pid":23730,"uri":"/crucible/0/regions/f70433db-9260-4cb7-8655-b2e0ab3333e8","method":"GET","req_id":"22dc2367-68e9-42c7-9404-188e6cca6134","remote_addr":"[fd00:1122:3344:101::a]:41817","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":42,"response_code":"200"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:23:09.701604307Z","hostname":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","pid":23730,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_b35cb2aa-7e4b-4820-82a1-1376e0cb1176/crucible/regions/f70433db-9260-4cb7-8655-b2e0ab3333e8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:23:09.713255648Z","hostname":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","pid":23730,"region":"f70433db-9260-4cb7-8655-b2e0ab3333e8","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:23:09.755116876Z","hostname":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","pid":23730,"uri":"/crucible/0/regions/f70433db-9260-4cb7-8655-b2e0ab3333e8","method":"GET","req_id":"42d7c47d-bf53-4e78-b83a-211a1fc87874","remote_addr":"[fd00:1122:3344:101::a]:41817","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":81,"response_code":"200"} {"msg":"region f70433db-9260-4cb7-8655-b2e0ab3333e8 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:23:09.78067916Z","hostname":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","pid":23730,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:23:09.859591849Z","hostname":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","pid":23730,"uri":"/crucible/0/regions/f70433db-9260-4cb7-8655-b2e0ab3333e8","method":"GET","req_id":"9f48ed23-85d3-4045-b8b9-c45c306f8c4c","remote_addr":"[fd00:1122:3344:101::a]:41817","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":58,"response_code":"200"}