[ Jan 27 09:31:31 Enabled. ] [ Jan 27 09:31:31 Rereading configuration. ] [ Jan 27 09:31:33 Rereading configuration. ] [ Jan 27 09:31:36 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_e4907ecd-72b0-4b94-88db-0ed5d4c83a8a/crucible -l [fd00:1122:3344:101::17]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"dataset: \"oxp_e4907ecd-72b0-4b94-88db-0ed5d4c83a8a/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:31:36.621262806Z","hostname":"oxz_crucible_5e4034d0-68b1-4e4d-8344-e580c4275afa","pid":24606} {"msg":"listen IP: [fd00:1122:3344:101::17]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:31:36.62229832Z","hostname":"oxz_crucible_5e4034d0-68b1-4e4d-8344-e580c4275afa","pid":24606} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:31:36.622309991Z","hostname":"oxz_crucible_5e4034d0-68b1-4e4d-8344-e580c4275afa","pid":24606} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:31:36.637988495Z","hostname":"oxz_crucible_5e4034d0-68b1-4e4d-8344-e580c4275afa","pid":24606,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:31:36.714261747Z","hostname":"oxz_crucible_5e4034d0-68b1-4e4d-8344-e580c4275afa","pid":24606,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:15.611376439Z","hostname":"oxz_crucible_5e4034d0-68b1-4e4d-8344-e580c4275afa","pid":24606,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:39658"} {"msg":"region c3f31cc5-5caa-49b9-9b37-43913ea70785 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:15.65660726Z","hostname":"oxz_crucible_5e4034d0-68b1-4e4d-8344-e580c4275afa","pid":24606,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:15.662232291Z","hostname":"oxz_crucible_5e4034d0-68b1-4e4d-8344-e580c4275afa","pid":24606,"uri":"/crucible/0/regions","method":"POST","req_id":"7c1ebd06-78eb-4729-b183-3c16bf5b0d27","remote_addr":"[fd00:1122:3344:101::b]:39658","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":19031,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:15.664855282Z","hostname":"oxz_crucible_5e4034d0-68b1-4e4d-8344-e580c4275afa","pid":24606,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_e4907ecd-72b0-4b94-88db-0ed5d4c83a8a/crucible/regions/c3f31cc5-5caa-49b9-9b37-43913ea70785","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:15.67511341Z","hostname":"oxz_crucible_5e4034d0-68b1-4e4d-8344-e580c4275afa","pid":24606,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_e4907ecd-72b0-4b94-88db-0ed5d4c83a8a/crucible/regions/c3f31cc5-5caa-49b9-9b37-43913ea70785","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:15.675148841Z","hostname":"oxz_crucible_5e4034d0-68b1-4e4d-8344-e580c4275afa","pid":24606,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"c3f31cc5-5caa-49b9-9b37-43913ea70785\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/c3f31cc5-5caa-49b9-9b37-43913ea70785\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:15.754689709Z","hostname":"oxz_crucible_5e4034d0-68b1-4e4d-8344-e580c4275afa","pid":24606,"region":"c3f31cc5-5caa-49b9-9b37-43913ea70785","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"c3f31cc5-5caa-49b9-9b37-43913ea70785\" \"--data\" \"/data/regions/c3f31cc5-5caa-49b9-9b37-43913ea70785\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:15.757054814Z","hostname":"oxz_crucible_5e4034d0-68b1-4e4d-8344-e580c4275afa","pid":24606,"region":"c3f31cc5-5caa-49b9-9b37-43913ea70785","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:15.916021517Z","hostname":"oxz_crucible_5e4034d0-68b1-4e4d-8344-e580c4275afa","pid":24606,"uri":"/crucible/0/regions","method":"POST","req_id":"a116880f-8039-4996-a1db-e909ca63ffff","remote_addr":"[fd00:1122:3344:101::b]:39658","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":87,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:16.068351766Z","hostname":"oxz_crucible_5e4034d0-68b1-4e4d-8344-e580c4275afa","pid":24606,"region":"c3f31cc5-5caa-49b9-9b37-43913ea70785","component":"worker"} {"msg":"region c3f31cc5-5caa-49b9-9b37-43913ea70785 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:16.068398117Z","hostname":"oxz_crucible_5e4034d0-68b1-4e4d-8344-e580c4275afa","pid":24606,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:16.068488189Z","hostname":"oxz_crucible_5e4034d0-68b1-4e4d-8344-e580c4275afa","pid":24606,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-c3f31cc5-5caa-49b9-9b37-43913ea70785","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:16.068878968Z","hostname":"oxz_crucible_5e4034d0-68b1-4e4d-8344-e580c4275afa","pid":24606,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-c3f31cc5-5caa-49b9-9b37-43913ea70785","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:16.078109763Z","hostname":"oxz_crucible_5e4034d0-68b1-4e4d-8344-e580c4275afa","pid":24606,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:16.100654687Z","hostname":"oxz_crucible_5e4034d0-68b1-4e4d-8344-e580c4275afa","pid":24606,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-c3f31cc5-5caa-49b9-9b37-43913ea70785","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:16.125902503Z","hostname":"oxz_crucible_5e4034d0-68b1-4e4d-8344-e580c4275afa","pid":24606,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c3f31cc5-5caa-49b9-9b37-43913ea70785","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:16.126763623Z","hostname":"oxz_crucible_5e4034d0-68b1-4e4d-8344-e580c4275afa","pid":24606,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:16.129080447Z","hostname":"oxz_crucible_5e4034d0-68b1-4e4d-8344-e580c4275afa","pid":24606,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:16.129136518Z","hostname":"oxz_crucible_5e4034d0-68b1-4e4d-8344-e580c4275afa","pid":24606,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:16.129184309Z","hostname":"oxz_crucible_5e4034d0-68b1-4e4d-8344-e580c4275afa","pid":24606,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:16.13696202Z","hostname":"oxz_crucible_5e4034d0-68b1-4e4d-8344-e580c4275afa","pid":24606,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:16.143510792Z","hostname":"oxz_crucible_5e4034d0-68b1-4e4d-8344-e580c4275afa","pid":24606,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:16.201999981Z","hostname":"oxz_crucible_5e4034d0-68b1-4e4d-8344-e580c4275afa","pid":24606,"uri":"/crucible/0/regions","method":"POST","req_id":"02f8fcf4-11f3-4ff5-93f2-9627bed95785","remote_addr":"[fd00:1122:3344:101::b]:39658","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":68,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:41.045056193Z","hostname":"oxz_crucible_5e4034d0-68b1-4e4d-8344-e580c4275afa","pid":24606,"uri":"/crucible/0/regions/c3f31cc5-5caa-49b9-9b37-43913ea70785","method":"GET","req_id":"67aaff0b-687b-437c-b46f-a916ce3ea179","remote_addr":"[fd00:1122:3344:101::b]:39658","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":60,"response_code":"200"} {"msg":"path is \"/data/regions/c3f31cc5-5caa-49b9-9b37-43913ea70785\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:41.047426299Z","hostname":"oxz_crucible_5e4034d0-68b1-4e4d-8344-e580c4275afa","pid":24606,"component":"datafile"} {"msg":"dataset is oxp_e4907ecd-72b0-4b94-88db-0ed5d4c83a8a/crucible/regions/c3f31cc5-5caa-49b9-9b37-43913ea70785","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:41.0595586Z","hostname":"oxz_crucible_5e4034d0-68b1-4e4d-8344-e580c4275afa","pid":24606,"component":"datafile"} {"msg":"region c3f31cc5-5caa-49b9-9b37-43913ea70785 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:41.06902751Z","hostname":"oxz_crucible_5e4034d0-68b1-4e4d-8344-e580c4275afa","pid":24606,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:41.069197964Z","hostname":"oxz_crucible_5e4034d0-68b1-4e4d-8344-e580c4275afa","pid":24606,"component":"worker"} {"msg":"disabling downstairs instance: downstairs-c3f31cc5-5caa-49b9-9b37-43913ea70785 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:41.070224488Z","hostname":"oxz_crucible_5e4034d0-68b1-4e4d-8344-e580c4275afa","pid":24606,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:41.071708542Z","hostname":"oxz_crucible_5e4034d0-68b1-4e4d-8344-e580c4275afa","pid":24606,"uri":"/crucible/0/regions/c3f31cc5-5caa-49b9-9b37-43913ea70785","method":"DELETE","req_id":"fa5d0047-d413-435b-b650-3428c745dd3c","remote_addr":"[fd00:1122:3344:101::b]:39658","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":24328,"response_code":"204"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:41.07419695Z","hostname":"oxz_crucible_5e4034d0-68b1-4e4d-8344-e580c4275afa","pid":24606,"uri":"/crucible/0/regions/c3f31cc5-5caa-49b9-9b37-43913ea70785","method":"GET","req_id":"d8045a09-16b6-4bbe-825e-796a2c000436","remote_addr":"[fd00:1122:3344:101::b]:39658","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":21,"response_code":"200"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:41.076082504Z","hostname":"oxz_crucible_5e4034d0-68b1-4e4d-8344-e580c4275afa","pid":24606,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_e4907ecd-72b0-4b94-88db-0ed5d4c83a8a/crucible/regions/c3f31cc5-5caa-49b9-9b37-43913ea70785\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:41.086953946Z","hostname":"oxz_crucible_5e4034d0-68b1-4e4d-8344-e580c4275afa","pid":24606,"region":"c3f31cc5-5caa-49b9-9b37-43913ea70785","component":"worker"} {"msg":"region c3f31cc5-5caa-49b9-9b37-43913ea70785 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:41.136926747Z","hostname":"oxz_crucible_5e4034d0-68b1-4e4d-8344-e580c4275afa","pid":24606,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:41.178175616Z","hostname":"oxz_crucible_5e4034d0-68b1-4e4d-8344-e580c4275afa","pid":24606,"uri":"/crucible/0/regions/c3f31cc5-5caa-49b9-9b37-43913ea70785","method":"GET","req_id":"2fce0703-d113-474c-9aeb-decb7d4959b5","remote_addr":"[fd00:1122:3344:101::b]:39658","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":85,"response_code":"200"}