[ May 15 07:12:15 Enabled. ]
[ May 15 07:12:15 Rereading configuration. ]
[ May 15 07:12:16 Rereading configuration. ]
[ May 15 07:12:19 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_08d4507c-e72e-4afd-9f03-5e28f4ad0909/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_08d4507c-e72e-4afd-9f03-5e28f4ad0909/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:19.485040293Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655}
{"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:19.49019683Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:19.490215231Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:19.516377935Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:19.630383797Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:45.27038612Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:56199"}
{"msg":"region 525ddce6-ef4e-4dce-9025-1ad93e926e9f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:45.335056905Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:45.339796064Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655,"uri":"/crucible/0/regions","method":"POST","req_id":"9870c0f9-5c84-45cb-abe8-21fd37787a56","remote_addr":"[fd00:1122:3344:101::a]:56199","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":15162,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:45.339824534Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_08d4507c-e72e-4afd-9f03-5e28f4ad0909/crucible/regions/525ddce6-ef4e-4dce-9025-1ad93e926e9f","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:45.351427046Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_08d4507c-e72e-4afd-9f03-5e28f4ad0909/crucible/regions/525ddce6-ef4e-4dce-9025-1ad93e926e9f","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:45.351459547Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"525ddce6-ef4e-4dce-9025-1ad93e926e9f\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/525ddce6-ef4e-4dce-9025-1ad93e926e9f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:45.432193666Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655,"region":"525ddce6-ef4e-4dce-9025-1ad93e926e9f","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"525ddce6-ef4e-4dce-9025-1ad93e926e9f\" \"--data\" \"/data/regions/525ddce6-ef4e-4dce-9025-1ad93e926e9f\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:45.434186388Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655,"region":"525ddce6-ef4e-4dce-9025-1ad93e926e9f","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:45.524221161Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655,"uri":"/crucible/0/regions","method":"POST","req_id":"3ff55630-65d0-4ffa-a131-43a5dd4d0d44","remote_addr":"[fd00:1122:3344:101::a]:56199","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":552,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:45.556432761Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655,"region":"525ddce6-ef4e-4dce-9025-1ad93e926e9f","component":"worker"}
{"msg":"region 525ddce6-ef4e-4dce-9025-1ad93e926e9f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:45.556464222Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:45.556564174Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-525ddce6-ef4e-4dce-9025-1ad93e926e9f","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:45.556969392Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-525ddce6-ef4e-4dce-9025-1ad93e926e9f","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:45.564298625Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:45.582706558Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-525ddce6-ef4e-4dce-9025-1ad93e926e9f","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:45.585513326Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/525ddce6-ef4e-4dce-9025-1ad93e926e9f","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:45.586294012Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:45.588286994Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:45.588354295Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:45.588396556Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:45.5972268Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:45.601995069Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:46.129939363Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655,"uri":"/crucible/0/regions","method":"POST","req_id":"f431ec75-0bde-405e-8680-75d3fc027f9e","remote_addr":"[fd00:1122:3344:101::a]:56199","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":74,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:46.434828176Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:64625"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:46.435649233Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655,"uri":"/crucible/0/regions/525ddce6-ef4e-4dce-9025-1ad93e926e9f","method":"GET","req_id":"114f5feb-f4a7-4dc7-8804-0e5dad3675ad","remote_addr":"[fd00:1122:3344:101::a]:64625","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":774,"response_code":"200"}
{"msg":"path is \"/data/regions/525ddce6-ef4e-4dce-9025-1ad93e926e9f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:46.435842227Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655,"component":"datafile"}
{"msg":"dataset is oxp_08d4507c-e72e-4afd-9f03-5e28f4ad0909/crucible/regions/525ddce6-ef4e-4dce-9025-1ad93e926e9f","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:46.446184452Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655,"component":"datafile"}
{"msg":"region 525ddce6-ef4e-4dce-9025-1ad93e926e9f state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:46.456342554Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:46.456474546Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:46.458423567Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655,"uri":"/crucible/0/regions/525ddce6-ef4e-4dce-9025-1ad93e926e9f","method":"DELETE","req_id":"4b6fbf61-8fad-4044-a956-88f54b245c6d","remote_addr":"[fd00:1122:3344:101::a]:64625","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":22602,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:46.45858657Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655,"uri":"/crucible/0/regions/525ddce6-ef4e-4dce-9025-1ad93e926e9f","method":"GET","req_id":"db12fe36-2277-46cd-bd54-8a692faf0654","remote_addr":"[fd00:1122:3344:101::a]:64625","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":22,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-525ddce6-ef4e-4dce-9025-1ad93e926e9f (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:46.459656443Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:46.464277799Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_08d4507c-e72e-4afd-9f03-5e28f4ad0909/crucible/regions/525ddce6-ef4e-4dce-9025-1ad93e926e9f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:46.47541585Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655,"region":"525ddce6-ef4e-4dce-9025-1ad93e926e9f","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:46.539986504Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655,"uri":"/crucible/0/regions/525ddce6-ef4e-4dce-9025-1ad93e926e9f","method":"GET","req_id":"f3cd1cca-957d-441d-a91f-067fd367393b","remote_addr":"[fd00:1122:3344:101::a]:64625","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":64,"response_code":"200"}
{"msg":"region 525ddce6-ef4e-4dce-9025-1ad93e926e9f state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:46.541673059Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-15T07:12:46.715307221Z","hostname":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","pid":22655,"uri":"/crucible/0/regions/525ddce6-ef4e-4dce-9025-1ad93e926e9f","method":"GET","req_id":"ef855708-9aca-4b06-9100-b193b8da562e","remote_addr":"[fd00:1122:3344:101::a]:64625","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":43,"response_code":"200"}