[ Nov 6 00:19:04 Enabled. ] [ Nov 6 00:19:04 Rereading configuration. ] [ Nov 6 00:19:06 Rereading configuration. ] [ Nov 6 00:19:08 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_25d63394-e252-4651-bd2a-cb7cb7ea5b3e/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"dataset: \"oxp_25d63394-e252-4651-bd2a-cb7cb7ea5b3e/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:19:08.956534535Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826} {"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:19:08.959688399Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:19:08.959701749Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:19:08.980380063Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:19:09.128051337Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:19:39.405187168Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:53859"} {"msg":"region ce2dfe53-d91c-48c8-94ba-b12c5332bc74 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:19:39.43153903Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:19:39.436694624Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"uri":"/crucible/0/regions","method":"POST","req_id":"6ce5ca69-6d75-4838-9fa4-edae4cdc4bf5","remote_addr":"[fd00:1122:3344:101::b]:53859","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":12639,"response_code":"200"} {"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:19:39.438804926Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"component":"worker"} {"msg":"zfs set reservation of 1342177280 for oxp_25d63394-e252-4651-bd2a-cb7cb7ea5b3e/crucible/regions/ce2dfe53-d91c-48c8-94ba-b12c5332bc74","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:19:39.448835934Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"component":"worker"} {"msg":"zfs set quota of 3221225472 for oxp_25d63394-e252-4651-bd2a-cb7cb7ea5b3e/crucible/regions/ce2dfe53-d91c-48c8-94ba-b12c5332bc74","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:19:39.448924337Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"ce2dfe53-d91c-48c8-94ba-b12c5332bc74\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/ce2dfe53-d91c-48c8-94ba-b12c5332bc74\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:19:39.532669603Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"region":"ce2dfe53-d91c-48c8-94ba-b12c5332bc74","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"ce2dfe53-d91c-48c8-94ba-b12c5332bc74\" \"--data\" \"/data/regions/ce2dfe53-d91c-48c8-94ba-b12c5332bc74\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:19:39.534809036Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"region":"ce2dfe53-d91c-48c8-94ba-b12c5332bc74","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:19:39.665205028Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"uri":"/crucible/0/regions","method":"POST","req_id":"543ce6be-150d-44e4-a6d2-98b5c727e804","remote_addr":"[fd00:1122:3344:101::b]:53859","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":76,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:19:39.845779659Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"region":"ce2dfe53-d91c-48c8-94ba-b12c5332bc74","component":"worker"} {"msg":"region ce2dfe53-d91c-48c8-94ba-b12c5332bc74 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:19:39.845861531Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:19:39.846101268Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-ce2dfe53-d91c-48c8-94ba-b12c5332bc74","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:19:39.84648775Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-ce2dfe53-d91c-48c8-94ba-b12c5332bc74","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:19:39.8528987Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:19:39.8640204Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-ce2dfe53-d91c-48c8-94ba-b12c5332bc74","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:19:39.868575196Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ce2dfe53-d91c-48c8-94ba-b12c5332bc74","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:19:39.869133472Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:19:39.869225385Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:19:39.869301257Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:19:39.869307347Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:19:39.87447294Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:19:39.880921862Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:19:40.26045115Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"uri":"/crucible/0/regions","method":"POST","req_id":"ca019416-33f9-4a51-a194-f79a4b5d58ea","remote_addr":"[fd00:1122:3344:101::b]:53859","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":66,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:19:40.863908506Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"uri":"/crucible/0/regions/ce2dfe53-d91c-48c8-94ba-b12c5332bc74","method":"GET","req_id":"8779e455-c718-4598-be76-79b7d8ec4bcc","remote_addr":"[fd00:1122:3344:101::b]:53859","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":100,"response_code":"200"} {"msg":"path is \"/data/regions/ce2dfe53-d91c-48c8-94ba-b12c5332bc74\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:19:40.868223044Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"component":"datafile"} {"msg":"dataset is oxp_25d63394-e252-4651-bd2a-cb7cb7ea5b3e/crucible/regions/ce2dfe53-d91c-48c8-94ba-b12c5332bc74","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:19:40.880291102Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"component":"datafile"} {"msg":"region ce2dfe53-d91c-48c8-94ba-b12c5332bc74 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:19:40.892078392Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:19:40.892876786Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"component":"worker"} {"msg":"disabling downstairs instance: downstairs-ce2dfe53-d91c-48c8-94ba-b12c5332bc74 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:19:40.893977479Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:19:40.894923867Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"uri":"/crucible/0/regions/ce2dfe53-d91c-48c8-94ba-b12c5332bc74","method":"DELETE","req_id":"0db358b9-6cf1-43bf-9804-99a14af94a57","remote_addr":"[fd00:1122:3344:101::b]:53859","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":26725,"response_code":"204"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:19:40.898092161Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"uri":"/crucible/0/regions/ce2dfe53-d91c-48c8-94ba-b12c5332bc74","method":"GET","req_id":"c1d43775-5dd0-41ba-93c5-ece7ca668cf2","remote_addr":"[fd00:1122:3344:101::b]:53859","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":42,"response_code":"200"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:19:40.898998718Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_25d63394-e252-4651-bd2a-cb7cb7ea5b3e/crucible/regions/ce2dfe53-d91c-48c8-94ba-b12c5332bc74\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:19:40.911573021Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"region":"ce2dfe53-d91c-48c8-94ba-b12c5332bc74","component":"worker"} {"msg":"region ce2dfe53-d91c-48c8-94ba-b12c5332bc74 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:19:41.00481922Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:19:41.036681816Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"uri":"/crucible/0/regions/ce2dfe53-d91c-48c8-94ba-b12c5332bc74","method":"GET","req_id":"697c2222-2855-4a42-b605-1a5abd6695b0","remote_addr":"[fd00:1122:3344:101::b]:53859","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":75,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:20:27.459565345Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:62537"} {"msg":"region 43010e51-c80e-466f-9547-6248987b74b5 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:20:27.459751581Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:20:27.459953847Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:20:27.459963237Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"uri":"/crucible/0/regions","method":"POST","req_id":"fd9b02cf-e003-437b-aaa4-87d624be8b11","remote_addr":"[fd00:1122:3344:101::a]:62537","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":245,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_25d63394-e252-4651-bd2a-cb7cb7ea5b3e/crucible/regions/43010e51-c80e-466f-9547-6248987b74b5","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:20:27.468397507Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_25d63394-e252-4651-bd2a-cb7cb7ea5b3e/crucible/regions/43010e51-c80e-466f-9547-6248987b74b5","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:20:27.468431418Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"43010e51-c80e-466f-9547-6248987b74b5\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/43010e51-c80e-466f-9547-6248987b74b5\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:20:27.52371523Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"region":"43010e51-c80e-466f-9547-6248987b74b5","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"43010e51-c80e-466f-9547-6248987b74b5\" \"--data\" \"/data/regions/43010e51-c80e-466f-9547-6248987b74b5\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:20:27.525560085Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"region":"43010e51-c80e-466f-9547-6248987b74b5","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:20:27.600886991Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"region":"43010e51-c80e-466f-9547-6248987b74b5","component":"worker"} {"msg":"region 43010e51-c80e-466f-9547-6248987b74b5 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:20:27.600918242Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:20:27.601014785Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-43010e51-c80e-466f-9547-6248987b74b5","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:20:27.602406476Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-43010e51-c80e-466f-9547-6248987b74b5","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:20:27.606410505Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:20:27.614827134Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-43010e51-c80e-466f-9547-6248987b74b5","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:20:27.618993978Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/43010e51-c80e-466f-9547-6248987b74b5","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:20:27.619304627Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:20:27.619358149Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:20:27.61939786Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:20:27.619437881Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:20:27.622719479Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:20:27.627960344Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-06T00:20:27.787708737Z","hostname":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","pid":25826,"uri":"/crucible/0/regions","method":"POST","req_id":"cc2daf83-e576-4a9d-8e23-4696b66414b4","remote_addr":"[fd00:1122:3344:101::a]:62537","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":68,"response_code":"200"}