[ May 8 22:47:57 Enabled. ] [ May 8 22:47:57 Rereading configuration. ] [ May 8 22:47:59 Rereading configuration. ] [ May 8 22:48:02 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_2a3f4f5a-b1a3-426c-884b-539eece88b9a/crucible -l [fd00:1122:3344:101::17]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:48:02.510562361Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378} {"msg":"dataset: \"oxp_2a3f4f5a-b1a3-426c-884b-539eece88b9a/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:48:02.511572225Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378} {"msg":"listen IP: [fd00:1122:3344:101::17]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:48:02.511590275Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:48:02.511599676Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:48:02.529775136Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:48:02.596401711Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:48:27.456880283Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:38247"} {"msg":"region a2506fac-db0f-4682-82a4-5826e5742081 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:48:27.494049422Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"component":"datafile"} {"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:48:27.495175999Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:48:27.495253941Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"uri":"/crucible/0/regions","method":"POST","req_id":"cd633c64-599e-45ae-b2ed-b3dac1aea1da","remote_addr":"[fd00:1122:3344:101::b]:38247","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":15067,"response_code":200} {"msg":"zfs set reservation of 1342177280 for oxp_2a3f4f5a-b1a3-426c-884b-539eece88b9a/crucible/regions/a2506fac-db0f-4682-82a4-5826e5742081","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:48:27.504965351Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"component":"worker"} {"msg":"zfs set quota of 3221225472 for oxp_2a3f4f5a-b1a3-426c-884b-539eece88b9a/crucible/regions/a2506fac-db0f-4682-82a4-5826e5742081","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:48:27.504984791Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"a2506fac-db0f-4682-82a4-5826e5742081\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/a2506fac-db0f-4682-82a4-5826e5742081\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:48:27.579289318Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"region":"a2506fac-db0f-4682-82a4-5826e5742081","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"a2506fac-db0f-4682-82a4-5826e5742081\" \"--data\" \"/data/regions/a2506fac-db0f-4682-82a4-5826e5742081\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:48:27.581329086Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"region":"a2506fac-db0f-4682-82a4-5826e5742081","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:48:27.714512836Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"region":"a2506fac-db0f-4682-82a4-5826e5742081","component":"worker"} {"msg":"region a2506fac-db0f-4682-82a4-5826e5742081 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:48:27.714578268Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:48:27.714772282Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-a2506fac-db0f-4682-82a4-5826e5742081","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:48:27.71509347Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-a2506fac-db0f-4682-82a4-5826e5742081","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:48:27.720495388Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:48:27.73453944Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a2506fac-db0f-4682-82a4-5826e5742081","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:48:27.736304171Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a2506fac-db0f-4682-82a4-5826e5742081","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:48:27.737037009Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:48:27.73710111Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:48:27.737150521Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:48:27.737206463Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:48:27.739747703Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:48:27.742720623Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:48:27.87874155Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"uri":"/crucible/0/regions","method":"POST","req_id":"d1f38e13-714b-4531-8649-5e615e793bf2","remote_addr":"[fd00:1122:3344:101::b]:38247","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":58,"response_code":200} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:48:28.743748765Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"uri":"/crucible/0/regions/a2506fac-db0f-4682-82a4-5826e5742081","method":"GET","req_id":"3c0685b7-9f82-4ef8-8aec-0a0170ba8ba3","remote_addr":"[fd00:1122:3344:101::b]:38247","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":95,"response_code":200} {"msg":"path is \"/data/regions/a2506fac-db0f-4682-82a4-5826e5742081\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:48:28.748388764Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"component":"datafile"} {"msg":"dataset is oxp_2a3f4f5a-b1a3-426c-884b-539eece88b9a/crucible/regions/a2506fac-db0f-4682-82a4-5826e5742081","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:48:28.755643696Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"component":"datafile"} {"msg":"region a2506fac-db0f-4682-82a4-5826e5742081 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:48:28.762352585Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:48:28.762537909Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:48:28.762549599Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"uri":"/crucible/0/regions/a2506fac-db0f-4682-82a4-5826e5742081","method":"DELETE","req_id":"120bd8c7-5df6-45e3-9d32-24c524264596","remote_addr":"[fd00:1122:3344:101::b]:38247","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":14211,"response_code":204} {"msg":"disabling downstairs instance: downstairs-a2506fac-db0f-4682-82a4-5826e5742081 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:48:28.763655655Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:48:28.765907829Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"uri":"/crucible/0/regions/a2506fac-db0f-4682-82a4-5826e5742081","method":"GET","req_id":"60965f9c-9260-4709-97ec-1845d63aecae","remote_addr":"[fd00:1122:3344:101::b]:38247","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":41,"response_code":200} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:48:28.766746458Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_2a3f4f5a-b1a3-426c-884b-539eece88b9a/crucible/regions/a2506fac-db0f-4682-82a4-5826e5742081\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:48:28.775600878Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"region":"a2506fac-db0f-4682-82a4-5826e5742081","component":"worker"} {"msg":"region a2506fac-db0f-4682-82a4-5826e5742081 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:48:28.843642017Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:48:28.890219448Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"uri":"/crucible/0/regions/a2506fac-db0f-4682-82a4-5826e5742081","method":"GET","req_id":"b8d5ff4e-1e7a-471c-99cb-0da17cfa21ae","remote_addr":"[fd00:1122:3344:101::b]:38247","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":56,"response_code":200} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:59:32.573673507Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:34424"} {"msg":"region c8407973-0b03-4715-872d-2bea0d15d8b5 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:59:32.573913253Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:59:32.574871035Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:59:32.574886536Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"uri":"/crucible/0/regions","method":"POST","req_id":"49eae69d-870c-4857-aa02-9510dff5c04e","remote_addr":"[fd00:1122:3344:101::b]:34424","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":1016,"response_code":200} {"msg":"zfs set reservation of 2684354560 for oxp_2a3f4f5a-b1a3-426c-884b-539eece88b9a/crucible/regions/c8407973-0b03-4715-872d-2bea0d15d8b5","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:59:32.582401083Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_2a3f4f5a-b1a3-426c-884b-539eece88b9a/crucible/regions/c8407973-0b03-4715-872d-2bea0d15d8b5","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:59:32.582422784Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"c8407973-0b03-4715-872d-2bea0d15d8b5\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/c8407973-0b03-4715-872d-2bea0d15d8b5\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:59:32.622338938Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"region":"c8407973-0b03-4715-872d-2bea0d15d8b5","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"c8407973-0b03-4715-872d-2bea0d15d8b5\" \"--data\" \"/data/regions/c8407973-0b03-4715-872d-2bea0d15d8b5\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:59:32.622390459Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"region":"c8407973-0b03-4715-872d-2bea0d15d8b5","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:59:32.681883326Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"region":"c8407973-0b03-4715-872d-2bea0d15d8b5","component":"worker"} {"msg":"region c8407973-0b03-4715-872d-2bea0d15d8b5 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:59:32.681996218Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:59:32.68204049Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-c8407973-0b03-4715-872d-2bea0d15d8b5","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:59:32.683095635Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-c8407973-0b03-4715-872d-2bea0d15d8b5","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:59:32.686767911Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:59:32.700376423Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-c8407973-0b03-4715-872d-2bea0d15d8b5","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:59:32.701897819Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c8407973-0b03-4715-872d-2bea0d15d8b5","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:59:32.702211467Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:59:32.702269618Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:59:32.702325109Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:59:32.702380251Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:59:32.704663645Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:59:32.707043931Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T22:59:32.757073514Z","hostname":"oxz_crucible_f5f70807-0457-436d-bfc7-6f35a46e193c","pid":21378,"uri":"/crucible/0/regions","method":"POST","req_id":"74761a07-344a-4b16-be18-8dc4237a6e36","remote_addr":"[fd00:1122:3344:101::b]:34424","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":63,"response_code":200}