[ Feb 23 14:12:32 Enabled. ] [ Feb 23 14:12:32 Rereading configuration. ] [ Feb 23 14:12:34 Rereading configuration. ] [ Feb 23 14:12:38 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_ee068547-8984-439c-b598-f02cc0127ab9/crucible -l [fd00:1122:3344:101::1a]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"dataset: \"oxp_ee068547-8984-439c-b598-f02cc0127ab9/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:38.596127737Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006} {"msg":"listen IP: [fd00:1122:3344:101::1a]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:38.597496861Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:38.597506661Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:38.618903131Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:38.738344327Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.596921456Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:45404"} {"msg":"region 3d0f1b69-6d8b-4a0a-8c11-1044de3a03fc state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.617167159Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.620615109Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"uri":"/crucible/0/regions","method":"POST","req_id":"90207a3f-3a2f-4a37-b2c6-1489d6e09ebf","remote_addr":"[fd00:1122:3344:101::b]:45404","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":10035,"response_code":"200"} {"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.620640569Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"component":"worker"} {"msg":"zfs set reservation of 1342177280 for oxp_ee068547-8984-439c-b598-f02cc0127ab9/crucible/regions/3d0f1b69-6d8b-4a0a-8c11-1044de3a03fc","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.629930835Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"component":"worker"} {"msg":"zfs set quota of 3221225472 for oxp_ee068547-8984-439c-b598-f02cc0127ab9/crucible/regions/3d0f1b69-6d8b-4a0a-8c11-1044de3a03fc","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.629989657Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"3d0f1b69-6d8b-4a0a-8c11-1044de3a03fc\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/3d0f1b69-6d8b-4a0a-8c11-1044de3a03fc\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.689652793Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"region":"3d0f1b69-6d8b-4a0a-8c11-1044de3a03fc","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"3d0f1b69-6d8b-4a0a-8c11-1044de3a03fc\" \"--data\" \"/data/regions/3d0f1b69-6d8b-4a0a-8c11-1044de3a03fc\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.691858263Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"region":"3d0f1b69-6d8b-4a0a-8c11-1044de3a03fc","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.758589624Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"uri":"/crucible/0/regions","method":"POST","req_id":"e64ae11c-6121-4d44-a575-0f0aca4f003a","remote_addr":"[fd00:1122:3344:101::b]:45404","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":76,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.811940529Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"region":"3d0f1b69-6d8b-4a0a-8c11-1044de3a03fc","component":"worker"} {"msg":"region 3d0f1b69-6d8b-4a0a-8c11-1044de3a03fc state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.812013302Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.812200938Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-3d0f1b69-6d8b-4a0a-8c11-1044de3a03fc","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.812614701Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-3d0f1b69-6d8b-4a0a-8c11-1044de3a03fc","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.816416812Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.829746235Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-3d0f1b69-6d8b-4a0a-8c11-1044de3a03fc","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.832566125Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3d0f1b69-6d8b-4a0a-8c11-1044de3a03fc","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.833300638Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.835643043Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.835705065Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.835746186Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.849250805Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:12:59.854697628Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:00.37394182Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"uri":"/crucible/0/regions","method":"POST","req_id":"c3347469-d272-4338-b4b1-5b33dc57c4a3","remote_addr":"[fd00:1122:3344:101::b]:45404","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":54,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:00.626446815Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"uri":"/crucible/0/regions/3d0f1b69-6d8b-4a0a-8c11-1044de3a03fc","method":"GET","req_id":"651341cb-0b7e-4e13-8ade-8240b732234b","remote_addr":"[fd00:1122:3344:101::b]:45404","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":55,"response_code":"200"} {"msg":"path is \"/data/regions/3d0f1b69-6d8b-4a0a-8c11-1044de3a03fc\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:00.629599195Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"component":"datafile"} {"msg":"dataset is oxp_ee068547-8984-439c-b598-f02cc0127ab9/crucible/regions/3d0f1b69-6d8b-4a0a-8c11-1044de3a03fc","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:00.640412259Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"component":"datafile"} {"msg":"region 3d0f1b69-6d8b-4a0a-8c11-1044de3a03fc state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:00.651347826Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:00.651580444Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"component":"worker"} {"msg":"disabling downstairs instance: downstairs-3d0f1b69-6d8b-4a0a-8c11-1044de3a03fc (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:00.652838364Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:00.653681641Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"uri":"/crucible/0/regions/3d0f1b69-6d8b-4a0a-8c11-1044de3a03fc","method":"DELETE","req_id":"24f5b52b-db2e-46a8-bd69-b47232fb36dc","remote_addr":"[fd00:1122:3344:101::b]:45404","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":24119,"response_code":"204"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:00.656922174Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"uri":"/crucible/0/regions/3d0f1b69-6d8b-4a0a-8c11-1044de3a03fc","method":"GET","req_id":"a3e8b3b1-3a61-4afb-bf66-8e8b9b335d30","remote_addr":"[fd00:1122:3344:101::b]:45404","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":29,"response_code":"200"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:00.657470551Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_ee068547-8984-439c-b598-f02cc0127ab9/crucible/regions/3d0f1b69-6d8b-4a0a-8c11-1044de3a03fc\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:00.669606057Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"region":"3d0f1b69-6d8b-4a0a-8c11-1044de3a03fc","component":"worker"} {"msg":"region 3d0f1b69-6d8b-4a0a-8c11-1044de3a03fc state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:00.751947424Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:00.785913163Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"uri":"/crucible/0/regions/3d0f1b69-6d8b-4a0a-8c11-1044de3a03fc","method":"GET","req_id":"455328ee-27eb-46ba-8262-9e41f0b0f350","remote_addr":"[fd00:1122:3344:101::b]:45404","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":70,"response_code":"200"} {"msg":"region 5faea10c-99a3-4c84-9c1e-536a91418802 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.417952732Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.420218844Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.420286726Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"uri":"/crucible/0/regions","method":"POST","req_id":"e1f4bd96-5f78-4a64-b28f-0970a27a6279","remote_addr":"[fd00:1122:3344:101::b]:45404","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":2320,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_ee068547-8984-439c-b598-f02cc0127ab9/crucible/regions/5faea10c-99a3-4c84-9c1e-536a91418802","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.429468298Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_ee068547-8984-439c-b598-f02cc0127ab9/crucible/regions/5faea10c-99a3-4c84-9c1e-536a91418802","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.429498259Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"5faea10c-99a3-4c84-9c1e-536a91418802\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/5faea10c-99a3-4c84-9c1e-536a91418802\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.496826509Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"region":"5faea10c-99a3-4c84-9c1e-536a91418802","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"5faea10c-99a3-4c84-9c1e-536a91418802\" \"--data\" \"/data/regions/5faea10c-99a3-4c84-9c1e-536a91418802\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.498552224Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"region":"5faea10c-99a3-4c84-9c1e-536a91418802","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.567817925Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"region":"5faea10c-99a3-4c84-9c1e-536a91418802","component":"worker"} {"msg":"region 5faea10c-99a3-4c84-9c1e-536a91418802 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.567892248Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.568122345Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-5faea10c-99a3-4c84-9c1e-536a91418802","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.569618732Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-5faea10c-99a3-4c84-9c1e-536a91418802","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.572440202Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.580615915Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-5faea10c-99a3-4c84-9c1e-536a91418802","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.583861675Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/5faea10c-99a3-4c84-9c1e-536a91418802","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.584203336Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.584268428Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.58431339Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.584365991Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.58842422Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.59313979Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-23T14:13:02.657526246Z","hostname":"oxz_crucible_91d28207-77aa-452a-aeb5-1ffd1304592a","pid":24006,"uri":"/crucible/0/regions","method":"POST","req_id":"c72a14e9-8074-4ec0-8744-80f3afcf595c","remote_addr":"[fd00:1122:3344:101::b]:45404","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":63,"response_code":"200"}