[ Jul 4 19:24:38 Enabled. ] [ Jul 4 19:24:38 Rereading configuration. ] [ Jul 4 19:24:39 Rereading configuration. ] [ Jul 4 19:24:42 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_9633ee26-fe12-4383-92d4-5ffe245edc8f/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"dataset: \"oxp_9633ee26-fe12-4383-92d4-5ffe245edc8f/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:24:42.985464361Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929} {"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:24:42.988147009Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:24:42.9881654Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:24:42.989794593Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:24:43.443908634Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:18.88775895Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:48341"} {"msg":"region f6791ebf-0846-4106-b932-cc90fffac40a state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:18.911633759Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:18.92153642Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"uri":"/crucible/0/regions","method":"POST","req_id":"6aa734a6-242f-4c97-90a2-87252298977b","remote_addr":"[fd00:1122:3344:101::c]:48341","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":19606,"response_code":"200"} {"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:18.921553971Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"component":"worker"} {"msg":"zfs set reservation of 1342177280 for oxp_9633ee26-fe12-4383-92d4-5ffe245edc8f/crucible/regions/f6791ebf-0846-4106-b932-cc90fffac40a","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:18.93486258Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"component":"worker"} {"msg":"zfs set quota of 3221225472 for oxp_9633ee26-fe12-4383-92d4-5ffe245edc8f/crucible/regions/f6791ebf-0846-4106-b932-cc90fffac40a","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:18.934921911Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"f6791ebf-0846-4106-b932-cc90fffac40a\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/f6791ebf-0846-4106-b932-cc90fffac40a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:19.000687626Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"region":"f6791ebf-0846-4106-b932-cc90fffac40a","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"f6791ebf-0846-4106-b932-cc90fffac40a\" \"--data\" \"/data/regions/f6791ebf-0846-4106-b932-cc90fffac40a\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:19.003137592Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"region":"f6791ebf-0846-4106-b932-cc90fffac40a","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:19.21952113Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"uri":"/crucible/0/regions","method":"POST","req_id":"f1db91f0-c960-4f17-870e-0d4eb4ae57a9","remote_addr":"[fd00:1122:3344:101::c]:48341","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":91,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:19.258964601Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"region":"f6791ebf-0846-4106-b932-cc90fffac40a","component":"worker"} {"msg":"region f6791ebf-0846-4106-b932-cc90fffac40a state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:19.259069413Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:19.259388438Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-f6791ebf-0846-4106-b932-cc90fffac40a","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:19.259895295Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f6791ebf-0846-4106-b932-cc90fffac40a","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:19.264085225Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:19.279440143Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f6791ebf-0846-4106-b932-cc90fffac40a","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:19.283260397Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f6791ebf-0846-4106-b932-cc90fffac40a","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:19.283977397Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:19.286458583Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:19.286517684Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:19.286556284Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:19.2974927Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:19.305267531Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:19.637803483Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"uri":"/crucible/0/regions","method":"POST","req_id":"4b371067-b00a-4ffd-902e-263bc070f376","remote_addr":"[fd00:1122:3344:101::c]:48341","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":528,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:19.839018666Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"uri":"/crucible/0/regions/f6791ebf-0846-4106-b932-cc90fffac40a","method":"GET","req_id":"ab271421-bdb9-42d1-a493-1223c13a7721","remote_addr":"[fd00:1122:3344:101::c]:48341","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":66,"response_code":"200"} {"msg":"path is \"/data/regions/f6791ebf-0846-4106-b932-cc90fffac40a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:19.841134096Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"component":"datafile"} {"msg":"dataset is oxp_9633ee26-fe12-4383-92d4-5ffe245edc8f/crucible/regions/f6791ebf-0846-4106-b932-cc90fffac40a","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:19.850334527Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"component":"datafile"} {"msg":"region f6791ebf-0846-4106-b932-cc90fffac40a state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:19.85899563Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:19.859228714Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:19.859309655Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"uri":"/crucible/0/regions/f6791ebf-0846-4106-b932-cc90fffac40a","method":"DELETE","req_id":"cde9d85b-d8b0-4e41-bdc3-64bc51207093","remote_addr":"[fd00:1122:3344:101::c]:48341","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":18234,"response_code":"204"} {"msg":"disabling downstairs instance: downstairs-f6791ebf-0846-4106-b932-cc90fffac40a (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:19.860493612Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:19.862162545Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"uri":"/crucible/0/regions/f6791ebf-0846-4106-b932-cc90fffac40a","method":"GET","req_id":"eac73c6b-82e2-4c18-a4c2-ba71782821bd","remote_addr":"[fd00:1122:3344:101::c]:48341","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":601,"response_code":"200"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:19.865155738Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_9633ee26-fe12-4383-92d4-5ffe245edc8f/crucible/regions/f6791ebf-0846-4106-b932-cc90fffac40a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:19.874862096Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"region":"f6791ebf-0846-4106-b932-cc90fffac40a","component":"worker"} {"msg":"region f6791ebf-0846-4106-b932-cc90fffac40a state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:19.931812727Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:20.002092607Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"uri":"/crucible/0/regions/f6791ebf-0846-4106-b932-cc90fffac40a","method":"GET","req_id":"fc9e9e23-a98c-4f8e-8a9d-a5cd7817d125","remote_addr":"[fd00:1122:3344:101::c]:48341","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":52,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:57.504388903Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:43896"} {"msg":"region 9c7860bc-23b6-4c4d-9d34-5cb2834aad1d state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:57.504543095Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:57.504709688Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:57.504719168Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"uri":"/crucible/0/regions","method":"POST","req_id":"3a225bde-7727-4c0a-ba62-dc1464595610","remote_addr":"[fd00:1122:3344:101::a]:43896","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":217,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_9633ee26-fe12-4383-92d4-5ffe245edc8f/crucible/regions/9c7860bc-23b6-4c4d-9d34-5cb2834aad1d","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:57.513751566Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_9633ee26-fe12-4383-92d4-5ffe245edc8f/crucible/regions/9c7860bc-23b6-4c4d-9d34-5cb2834aad1d","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:57.513776687Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"9c7860bc-23b6-4c4d-9d34-5cb2834aad1d\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/9c7860bc-23b6-4c4d-9d34-5cb2834aad1d\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:57.563965211Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"region":"9c7860bc-23b6-4c4d-9d34-5cb2834aad1d","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"9c7860bc-23b6-4c4d-9d34-5cb2834aad1d\" \"--data\" \"/data/regions/9c7860bc-23b6-4c4d-9d34-5cb2834aad1d\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:57.564018282Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"region":"9c7860bc-23b6-4c4d-9d34-5cb2834aad1d","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:57.62644326Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"region":"9c7860bc-23b6-4c4d-9d34-5cb2834aad1d","component":"worker"} {"msg":"region 9c7860bc-23b6-4c4d-9d34-5cb2834aad1d state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:57.62647947Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:57.626570382Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-9c7860bc-23b6-4c4d-9d34-5cb2834aad1d","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:57.628128574Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-9c7860bc-23b6-4c4d-9d34-5cb2834aad1d","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:57.630923894Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:57.638730785Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-9c7860bc-23b6-4c4d-9d34-5cb2834aad1d","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:57.641338442Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/9c7860bc-23b6-4c4d-9d34-5cb2834aad1d","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:57.641701857Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:57.641750098Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:57.641791858Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:57.641828329Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:57.645366139Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:57.64967054Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-04T19:25:57.714790787Z","hostname":"oxz_crucible_13871b2d-281d-4030-ba0c-727e430c26f5","pid":21929,"uri":"/crucible/0/regions","method":"POST","req_id":"bbcb280d-1ea4-40a1-9d74-dfca1cb17c96","remote_addr":"[fd00:1122:3344:101::a]:43896","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":59,"response_code":"200"}