[ Oct 24 18:54:35 Enabled. ]
[ Oct 24 18:54:35 Rereading configuration. ]
[ Oct 24 18:54:39 Rereading configuration. ]
[ Oct 24 18:54:43 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_fa33182c-5386-4788-bb69-fed5eea70499/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_fa33182c-5386-4788-bb69-fed5eea70499/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:54:43.237739109Z","hostname":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","pid":24038}
{"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:54:43.239567127Z","hostname":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","pid":24038}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:54:43.239577667Z","hostname":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","pid":24038}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:54:43.267591139Z","hostname":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","pid":24038,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:54:43.469869317Z","hostname":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","pid":24038,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.094324239Z","hostname":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","pid":24038,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:60462"}
{"msg":"region f734e6e4-4b33-4c57-9557-555b7e53bc54 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.114573891Z","hostname":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","pid":24038,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.120294889Z","hostname":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","pid":24038,"uri":"/crucible/0/regions","method":"POST","req_id":"716f9ab1-6ffc-43d4-9f79-abaf0a6a7f9f","remote_addr":"[fd00:1122:3344:101::c]:60462","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":9761,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.120314739Z","hostname":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","pid":24038,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_fa33182c-5386-4788-bb69-fed5eea70499/crucible/regions/f734e6e4-4b33-4c57-9557-555b7e53bc54","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.129884877Z","hostname":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","pid":24038,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_fa33182c-5386-4788-bb69-fed5eea70499/crucible/regions/f734e6e4-4b33-4c57-9557-555b7e53bc54","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.129936268Z","hostname":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","pid":24038,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"f734e6e4-4b33-4c57-9557-555b7e53bc54\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/f734e6e4-4b33-4c57-9557-555b7e53bc54\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.201027923Z","hostname":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","pid":24038,"region":"f734e6e4-4b33-4c57-9557-555b7e53bc54","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"f734e6e4-4b33-4c57-9557-555b7e53bc54\" \"--data\" \"/data/regions/f734e6e4-4b33-4c57-9557-555b7e53bc54\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.203034365Z","hostname":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","pid":24038,"region":"f734e6e4-4b33-4c57-9557-555b7e53bc54","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.350390426Z","hostname":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","pid":24038,"region":"f734e6e4-4b33-4c57-9557-555b7e53bc54","component":"worker"}
{"msg":"region f734e6e4-4b33-4c57-9557-555b7e53bc54 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.350471327Z","hostname":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","pid":24038,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.35067031Z","hostname":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","pid":24038,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-f734e6e4-4b33-4c57-9557-555b7e53bc54","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.351080197Z","hostname":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","pid":24038,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f734e6e4-4b33-4c57-9557-555b7e53bc54","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.355303002Z","hostname":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","pid":24038,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.366147669Z","hostname":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","pid":24038,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f734e6e4-4b33-4c57-9557-555b7e53bc54","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.369143575Z","hostname":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","pid":24038,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f734e6e4-4b33-4c57-9557-555b7e53bc54","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.369691883Z","hostname":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","pid":24038,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.369751395Z","hostname":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","pid":24038,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.369789995Z","hostname":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","pid":24038,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.369839156Z","hostname":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","pid":24038,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.373883038Z","hostname":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","pid":24038,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.377861289Z","hostname":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","pid":24038,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:06.397789597Z","hostname":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","pid":24038,"uri":"/crucible/0/regions","method":"POST","req_id":"b8630282-b526-48cb-b90e-e6ee74e38839","remote_addr":"[fd00:1122:3344:101::c]:60462","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":61,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:07.273968893Z","hostname":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","pid":24038,"uri":"/crucible/0/regions/f734e6e4-4b33-4c57-9557-555b7e53bc54","method":"GET","req_id":"2ec84e2f-d751-4ddd-a6a6-54ad84a03253","remote_addr":"[fd00:1122:3344:101::c]:60462","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":86,"response_code":"200"}
{"msg":"path is \"/data/regions/f734e6e4-4b33-4c57-9557-555b7e53bc54\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:07.276667555Z","hostname":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","pid":24038,"component":"datafile"}
{"msg":"dataset is oxp_fa33182c-5386-4788-bb69-fed5eea70499/crucible/regions/f734e6e4-4b33-4c57-9557-555b7e53bc54","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:07.287799256Z","hostname":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","pid":24038,"component":"datafile"}
{"msg":"region f734e6e4-4b33-4c57-9557-555b7e53bc54 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:07.298014064Z","hostname":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","pid":24038,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:07.298216717Z","hostname":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","pid":24038,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:07.298299998Z","hostname":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","pid":24038,"uri":"/crucible/0/regions/f734e6e4-4b33-4c57-9557-555b7e53bc54","method":"DELETE","req_id":"7609d644-ec96-4c2c-a2c6-a6b4ddd6d448","remote_addr":"[fd00:1122:3344:101::c]:60462","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":21665,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-f734e6e4-4b33-4c57-9557-555b7e53bc54 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:07.299278753Z","hostname":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","pid":24038,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:07.301069471Z","hostname":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","pid":24038,"uri":"/crucible/0/regions/f734e6e4-4b33-4c57-9557-555b7e53bc54","method":"GET","req_id":"b88012a8-36f2-46bc-8dbc-22f60760ac95","remote_addr":"[fd00:1122:3344:101::c]:60462","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":50,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:07.305100083Z","hostname":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","pid":24038,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_fa33182c-5386-4788-bb69-fed5eea70499/crucible/regions/f734e6e4-4b33-4c57-9557-555b7e53bc54\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:07.31728029Z","hostname":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","pid":24038,"region":"f734e6e4-4b33-4c57-9557-555b7e53bc54","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:07.397404076Z","hostname":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","pid":24038,"uri":"/crucible/0/regions/f734e6e4-4b33-4c57-9557-555b7e53bc54","method":"GET","req_id":"be768c62-6c48-4ee5-a788-5b211bf3b207","remote_addr":"[fd00:1122:3344:101::c]:60462","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":89,"response_code":"200"}
{"msg":"region f734e6e4-4b33-4c57-9557-555b7e53bc54 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:07.401304956Z","hostname":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","pid":24038,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-24T18:55:07.545072352Z","hostname":"oxz_crucible_b27d58f3-b052-4217-923a-62b3896e867a","pid":24038,"uri":"/crucible/0/regions/f734e6e4-4b33-4c57-9557-555b7e53bc54","method":"GET","req_id":"8d1f4686-4dab-4f64-993c-edf25439ac03","remote_addr":"[fd00:1122:3344:101::c]:60462","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":48,"response_code":"200"}