[ Oct 14 10:09:54 Enabled. ]
[ Oct 14 10:09:54 Rereading configuration. ]
[ Oct 14 10:09:57 Rereading configuration. ]
[ Oct 14 10:10:05 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_679d0661-57e9-4d21-94ff-f3b3f439ca94/crucible -l [fd00:1122:3344:101::1a]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_679d0661-57e9-4d21-94ff-f3b3f439ca94/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:10:05.400506938Z","hostname":"oxz_crucible_fd456a42-5008-4c2e-b191-9689dc3d2b71","pid":22465}
{"msg":"listen IP: [fd00:1122:3344:101::1a]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:10:05.402390167Z","hostname":"oxz_crucible_fd456a42-5008-4c2e-b191-9689dc3d2b71","pid":22465}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:10:05.402402517Z","hostname":"oxz_crucible_fd456a42-5008-4c2e-b191-9689dc3d2b71","pid":22465}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:10:05.414686982Z","hostname":"oxz_crucible_fd456a42-5008-4c2e-b191-9689dc3d2b71","pid":22465,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:10:05.605435087Z","hostname":"oxz_crucible_fd456a42-5008-4c2e-b191-9689dc3d2b71","pid":22465,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.366915699Z","hostname":"oxz_crucible_fd456a42-5008-4c2e-b191-9689dc3d2b71","pid":22465,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:42673"}
{"msg":"region 36ce56ca-268d-4433-bb28-f6aa154ce280 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.410124007Z","hostname":"oxz_crucible_fd456a42-5008-4c2e-b191-9689dc3d2b71","pid":22465,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.415597141Z","hostname":"oxz_crucible_fd456a42-5008-4c2e-b191-9689dc3d2b71","pid":22465,"uri":"/crucible/0/regions","method":"POST","req_id":"97147617-1bb9-446a-b96a-03032f108757","remote_addr":"[fd00:1122:3344:101::a]:42673","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":16412,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.420094015Z","hostname":"oxz_crucible_fd456a42-5008-4c2e-b191-9689dc3d2b71","pid":22465,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_679d0661-57e9-4d21-94ff-f3b3f439ca94/crucible/regions/36ce56ca-268d-4433-bb28-f6aa154ce280","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.430191474Z","hostname":"oxz_crucible_fd456a42-5008-4c2e-b191-9689dc3d2b71","pid":22465,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_679d0661-57e9-4d21-94ff-f3b3f439ca94/crucible/regions/36ce56ca-268d-4433-bb28-f6aa154ce280","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.430221765Z","hostname":"oxz_crucible_fd456a42-5008-4c2e-b191-9689dc3d2b71","pid":22465,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"36ce56ca-268d-4433-bb28-f6aa154ce280\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/36ce56ca-268d-4433-bb28-f6aa154ce280\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.477362455Z","hostname":"oxz_crucible_fd456a42-5008-4c2e-b191-9689dc3d2b71","pid":22465,"region":"36ce56ca-268d-4433-bb28-f6aa154ce280","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"36ce56ca-268d-4433-bb28-f6aa154ce280\" \"--data\" \"/data/regions/36ce56ca-268d-4433-bb28-f6aa154ce280\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.482198786Z","hostname":"oxz_crucible_fd456a42-5008-4c2e-b191-9689dc3d2b71","pid":22465,"region":"36ce56ca-268d-4433-bb28-f6aa154ce280","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.665795072Z","hostname":"oxz_crucible_fd456a42-5008-4c2e-b191-9689dc3d2b71","pid":22465,"uri":"/crucible/0/regions","method":"POST","req_id":"b106a312-2c88-4a88-8aaf-c34d68347f22","remote_addr":"[fd00:1122:3344:101::a]:42673","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":83,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.859168141Z","hostname":"oxz_crucible_fd456a42-5008-4c2e-b191-9689dc3d2b71","pid":22465,"region":"36ce56ca-268d-4433-bb28-f6aa154ce280","component":"worker"}
{"msg":"region 36ce56ca-268d-4433-bb28-f6aa154ce280 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.859224362Z","hostname":"oxz_crucible_fd456a42-5008-4c2e-b191-9689dc3d2b71","pid":22465,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.859321994Z","hostname":"oxz_crucible_fd456a42-5008-4c2e-b191-9689dc3d2b71","pid":22465,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-36ce56ca-268d-4433-bb28-f6aa154ce280","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.862086252Z","hostname":"oxz_crucible_fd456a42-5008-4c2e-b191-9689dc3d2b71","pid":22465,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-36ce56ca-268d-4433-bb28-f6aa154ce280","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.867858582Z","hostname":"oxz_crucible_fd456a42-5008-4c2e-b191-9689dc3d2b71","pid":22465,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.890538653Z","hostname":"oxz_crucible_fd456a42-5008-4c2e-b191-9689dc3d2b71","pid":22465,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-36ce56ca-268d-4433-bb28-f6aa154ce280","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.896072198Z","hostname":"oxz_crucible_fd456a42-5008-4c2e-b191-9689dc3d2b71","pid":22465,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/36ce56ca-268d-4433-bb28-f6aa154ce280","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.90531136Z","hostname":"oxz_crucible_fd456a42-5008-4c2e-b191-9689dc3d2b71","pid":22465,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.905379991Z","hostname":"oxz_crucible_fd456a42-5008-4c2e-b191-9689dc3d2b71","pid":22465,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.905421052Z","hostname":"oxz_crucible_fd456a42-5008-4c2e-b191-9689dc3d2b71","pid":22465,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.905457233Z","hostname":"oxz_crucible_fd456a42-5008-4c2e-b191-9689dc3d2b71","pid":22465,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.911534759Z","hostname":"oxz_crucible_fd456a42-5008-4c2e-b191-9689dc3d2b71","pid":22465,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:21.919280291Z","hostname":"oxz_crucible_fd456a42-5008-4c2e-b191-9689dc3d2b71","pid":22465,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:22.314335922Z","hostname":"oxz_crucible_fd456a42-5008-4c2e-b191-9689dc3d2b71","pid":22465,"uri":"/crucible/0/regions","method":"POST","req_id":"f0eaa773-db65-46bd-90c9-d0806c86c216","remote_addr":"[fd00:1122:3344:101::a]:42673","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":59,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:47.551516117Z","hostname":"oxz_crucible_fd456a42-5008-4c2e-b191-9689dc3d2b71","pid":22465,"uri":"/crucible/0/regions/36ce56ca-268d-4433-bb28-f6aa154ce280","method":"GET","req_id":"a06514f8-a6d5-442c-947d-70aa8306c10d","remote_addr":"[fd00:1122:3344:101::a]:42673","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":117,"response_code":"200"}
{"msg":"path is \"/data/regions/36ce56ca-268d-4433-bb28-f6aa154ce280\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:47.554521109Z","hostname":"oxz_crucible_fd456a42-5008-4c2e-b191-9689dc3d2b71","pid":22465,"component":"datafile"}
{"msg":"dataset is oxp_679d0661-57e9-4d21-94ff-f3b3f439ca94/crucible/regions/36ce56ca-268d-4433-bb28-f6aa154ce280","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:47.566778724Z","hostname":"oxz_crucible_fd456a42-5008-4c2e-b191-9689dc3d2b71","pid":22465,"component":"datafile"}
{"msg":"region 36ce56ca-268d-4433-bb28-f6aa154ce280 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:47.576503976Z","hostname":"oxz_crucible_fd456a42-5008-4c2e-b191-9689dc3d2b71","pid":22465,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:47.5766952Z","hostname":"oxz_crucible_fd456a42-5008-4c2e-b191-9689dc3d2b71","pid":22465,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:47.576788412Z","hostname":"oxz_crucible_fd456a42-5008-4c2e-b191-9689dc3d2b71","pid":22465,"uri":"/crucible/0/regions/36ce56ca-268d-4433-bb28-f6aa154ce280","method":"DELETE","req_id":"6d72f3d9-5c76-4205-bcbc-61d571297b73","remote_addr":"[fd00:1122:3344:101::a]:42673","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":22295,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-36ce56ca-268d-4433-bb28-f6aa154ce280 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:47.577801703Z","hostname":"oxz_crucible_fd456a42-5008-4c2e-b191-9689dc3d2b71","pid":22465,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:47.58199273Z","hostname":"oxz_crucible_fd456a42-5008-4c2e-b191-9689dc3d2b71","pid":22465,"uri":"/crucible/0/regions/36ce56ca-268d-4433-bb28-f6aa154ce280","method":"GET","req_id":"79b8e865-0cda-450a-8930-75f0ab6cac85","remote_addr":"[fd00:1122:3344:101::a]:42673","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":660,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:47.582158984Z","hostname":"oxz_crucible_fd456a42-5008-4c2e-b191-9689dc3d2b71","pid":22465,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_679d0661-57e9-4d21-94ff-f3b3f439ca94/crucible/regions/36ce56ca-268d-4433-bb28-f6aa154ce280\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:47.592600161Z","hostname":"oxz_crucible_fd456a42-5008-4c2e-b191-9689dc3d2b71","pid":22465,"region":"36ce56ca-268d-4433-bb28-f6aa154ce280","component":"worker"}
{"msg":"region 36ce56ca-268d-4433-bb28-f6aa154ce280 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:47.641112999Z","hostname":"oxz_crucible_fd456a42-5008-4c2e-b191-9689dc3d2b71","pid":22465,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-14T10:14:47.721565951Z","hostname":"oxz_crucible_fd456a42-5008-4c2e-b191-9689dc3d2b71","pid":22465,"uri":"/crucible/0/regions/36ce56ca-268d-4433-bb28-f6aa154ce280","method":"GET","req_id":"b66dbcab-07d7-40e1-be37-012c12db6dfe","remote_addr":"[fd00:1122:3344:101::a]:42673","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":97,"response_code":"200"}