[ Oct 18 17:56:19 Enabled. ] [ Oct 18 17:56:19 Rereading configuration. ] [ Oct 18 17:56:21 Rereading configuration. ] [ Oct 18 17:56:22 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_883d1cbf-e1f8-4c86-b18f-d626feb923fd/crucible -l [fd00:1122:3344:101::1a]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"dataset: \"oxp_883d1cbf-e1f8-4c86-b18f-d626feb923fd/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:56:23.060500955Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824} {"msg":"listen IP: [fd00:1122:3344:101::1a]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:56:23.065675133Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:56:23.065690073Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:56:23.080539241Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:56:23.222529436Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:56:59.219578693Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:62705"} {"msg":"region d52c189e-5059-497c-8d7f-cf8ac353b2cd state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:56:59.259469982Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:56:59.26863149Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"uri":"/crucible/0/regions","method":"POST","req_id":"3b7b72a1-506e-490b-bad5-b97b255087ec","remote_addr":"[fd00:1122:3344:101::a]:62705","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":18199,"response_code":"200"} {"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:56:59.268659081Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"component":"worker"} {"msg":"zfs set reservation of 1342177280 for oxp_883d1cbf-e1f8-4c86-b18f-d626feb923fd/crucible/regions/d52c189e-5059-497c-8d7f-cf8ac353b2cd","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:56:59.278156607Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"component":"worker"} {"msg":"zfs set quota of 3221225472 for oxp_883d1cbf-e1f8-4c86-b18f-d626feb923fd/crucible/regions/d52c189e-5059-497c-8d7f-cf8ac353b2cd","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:56:59.278187998Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"d52c189e-5059-497c-8d7f-cf8ac353b2cd\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/d52c189e-5059-497c-8d7f-cf8ac353b2cd\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:56:59.334522702Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"region":"d52c189e-5059-497c-8d7f-cf8ac353b2cd","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"d52c189e-5059-497c-8d7f-cf8ac353b2cd\" \"--data\" \"/data/regions/d52c189e-5059-497c-8d7f-cf8ac353b2cd\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:56:59.33928854Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"region":"d52c189e-5059-497c-8d7f-cf8ac353b2cd","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:56:59.627862513Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"uri":"/crucible/0/regions","method":"POST","req_id":"364d0c12-82fb-4227-9a41-fa518a36c007","remote_addr":"[fd00:1122:3344:101::a]:62705","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":83,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:56:59.645256759Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"region":"d52c189e-5059-497c-8d7f-cf8ac353b2cd","component":"worker"} {"msg":"region d52c189e-5059-497c-8d7f-cf8ac353b2cd state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:56:59.64530911Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:56:59.645558366Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-d52c189e-5059-497c-8d7f-cf8ac353b2cd","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:56:59.648177476Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-d52c189e-5059-497c-8d7f-cf8ac353b2cd","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:56:59.654016839Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:56:59.668175971Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-d52c189e-5059-497c-8d7f-cf8ac353b2cd","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:56:59.679752875Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/d52c189e-5059-497c-8d7f-cf8ac353b2cd","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:56:59.680288497Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:56:59.680356039Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:56:59.68040025Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:56:59.680438511Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:56:59.684553524Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:56:59.690223314Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:56:59.963318654Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"uri":"/crucible/0/regions","method":"POST","req_id":"7aa2250f-89ff-420c-bea4-fae94e9c1b7f","remote_addr":"[fd00:1122:3344:101::a]:62705","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":69,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:57:00.368048514Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"uri":"/crucible/0/regions/d52c189e-5059-497c-8d7f-cf8ac353b2cd","method":"GET","req_id":"e4da0ead-dd78-4d0f-b310-335c57e80cec","remote_addr":"[fd00:1122:3344:101::a]:62705","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":82,"response_code":"200"} {"msg":"path is \"/data/regions/d52c189e-5059-497c-8d7f-cf8ac353b2cd\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:57:00.371202986Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"component":"datafile"} {"msg":"dataset is oxp_883d1cbf-e1f8-4c86-b18f-d626feb923fd/crucible/regions/d52c189e-5059-497c-8d7f-cf8ac353b2cd","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:57:00.381988662Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"component":"datafile"} {"msg":"region d52c189e-5059-497c-8d7f-cf8ac353b2cd state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:57:00.392108562Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:57:00.392325997Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:57:00.392406019Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"uri":"/crucible/0/regions/d52c189e-5059-497c-8d7f-cf8ac353b2cd","method":"DELETE","req_id":"53d57d2e-73cc-4ea8-871b-8d8e8d1c7c19","remote_addr":"[fd00:1122:3344:101::a]:62705","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":21233,"response_code":"204"} {"msg":"disabling downstairs instance: downstairs-d52c189e-5059-497c-8d7f-cf8ac353b2cd (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:57:00.393545505Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:57:00.39596587Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"uri":"/crucible/0/regions/d52c189e-5059-497c-8d7f-cf8ac353b2cd","method":"GET","req_id":"e0fc1ff0-27e8-420e-ba5e-284cb318b65d","remote_addr":"[fd00:1122:3344:101::a]:62705","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":32,"response_code":"200"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:57:00.400617846Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_883d1cbf-e1f8-4c86-b18f-d626feb923fd/crucible/regions/d52c189e-5059-497c-8d7f-cf8ac353b2cd\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:57:00.412532077Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"region":"d52c189e-5059-497c-8d7f-cf8ac353b2cd","component":"worker"} {"msg":"region d52c189e-5059-497c-8d7f-cf8ac353b2cd state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:57:00.485782616Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:57:00.505983646Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"uri":"/crucible/0/regions/d52c189e-5059-497c-8d7f-cf8ac353b2cd","method":"GET","req_id":"a9a79ed6-c015-4189-ae75-5ed482f47949","remote_addr":"[fd00:1122:3344:101::a]:62705","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":68,"response_code":"200"} {"msg":"region d90bf3df-f84b-4e1a-b974-984c22788141 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:57:02.330427634Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:57:02.330602118Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:57:02.330642639Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"uri":"/crucible/0/regions","method":"POST","req_id":"3fa59c66-2731-46d3-b34b-c20d4f0eb847","remote_addr":"[fd00:1122:3344:101::a]:62705","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":242,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_883d1cbf-e1f8-4c86-b18f-d626feb923fd/crucible/regions/d90bf3df-f84b-4e1a-b974-984c22788141","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:57:02.338513439Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_883d1cbf-e1f8-4c86-b18f-d626feb923fd/crucible/regions/d90bf3df-f84b-4e1a-b974-984c22788141","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:57:02.338548359Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"d90bf3df-f84b-4e1a-b974-984c22788141\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/d90bf3df-f84b-4e1a-b974-984c22788141\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:57:02.41408713Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"region":"d90bf3df-f84b-4e1a-b974-984c22788141","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"d90bf3df-f84b-4e1a-b974-984c22788141\" \"--data\" \"/data/regions/d90bf3df-f84b-4e1a-b974-984c22788141\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:57:02.41585748Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"region":"d90bf3df-f84b-4e1a-b974-984c22788141","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:57:02.485267351Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"region":"d90bf3df-f84b-4e1a-b974-984c22788141","component":"worker"} {"msg":"region d90bf3df-f84b-4e1a-b974-984c22788141 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:57:02.485338583Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:57:02.485568788Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-d90bf3df-f84b-4e1a-b974-984c22788141","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:57:02.487100393Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-d90bf3df-f84b-4e1a-b974-984c22788141","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:57:02.491112945Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:57:02.499193758Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-d90bf3df-f84b-4e1a-b974-984c22788141","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:57:02.505740718Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/d90bf3df-f84b-4e1a-b974-984c22788141","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:57:02.505947692Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:57:02.506002814Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:57:02.506044245Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:57:02.506081795Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:57:02.510342343Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:57:02.51528517Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T17:57:02.644001617Z","hostname":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","pid":21824,"uri":"/crucible/0/regions","method":"POST","req_id":"ec61af99-ab1a-4f48-bf2f-61c0747cabdd","remote_addr":"[fd00:1122:3344:101::a]:62705","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":77,"response_code":"200"}