can quats do anything?

Better version including matrix conversions. This also has a couple of fixes to enforce extension classes loading order and accounting for userdata metatables __type member in type.is, which may have been renamed from __instanceDict, either table is unique per userdata subclass/metadata so it makes a good type id for userdata objects.

extensions = {}
extensions.all = {}
extensions.enabled = {}
extensions.hook_base_nil = function() end
extensions.hook = function(base,func)
    if func == nil then
        func = base
        base = nil
    end
    base = base or extensions.hook_base_nil
    return function(...)
        return func(base,...)
    end
end

extensions.register = function(extension)
    local extensionType = type(extension)
    assert(extensionType == "function" or extensionType == "table")
    table.insert(extensions.all, extension)
end

Extension = class()
function Extension:init()
end

extensions.extension = function(base)
    base = base or Extension
    local ext = class(base)
    extensions.register(ext)
    return ext
end

extensions.setup = function(...)
    local args = {...}
    local enabled = {}
    if #args == 0 then
        for i,v in ipairs(extensions.all) do
            table.insert(enabled, v)
        end
    elseif #args == 1 then
        local arg = args[1]
        if type(arg) == "table" then
            local exclude = arg.exclude
            excluded = {}
            if exclude then
                for i,ext in ipairs(exclude) do
                    excluded[ext] = ext
                end
            end
            local include =  arg.include or (arg.exclude == nil and arg)
            included = {}
            for i,ext in ipairs(include) do
                included[ext]=ext
            end
            for i,ext in ipairs(extensions.all) do
                if excluded[ext] == nil or included[ext] ~= nil then
                    table.insert(enabled, ext)
                end
            end
        end
    else
        for i,ext in ipairs(args) do
            table.insert(enabled,ext)
        end
    end
    for i,extension in ipairs(enabled) do
        local extensionType = type(extension)
        if extensionType == "function" then
            extension()
        elseif extensionType == "table" and extension.setup then
            local ext = extension()
            ext:setup()
        else
            error("Unsupported extension")
        end
    end
    extensions.enabled = enabled
end

TypeExtensions = extensions.extension()
function TypeExtensions:setup()
    local typeFunc = type
    local typeTable = {}
    local is_class = function(value)
        return typeof(value) == "class"
    end
    typeTable.is_class = is_class
    typeTable.is = function(value, typ)
        local t = typeFunc(value)
        if typ == "class" then
            return is_class(value) 
        end
        local tt = typeFunc(typ)
        if (t == "table" or t == "userdata") and tt == "table" then
            -- easier built-in instance check
            if t == "table" and value.isInstanceOf and value:isInstanceOf(typ) then
                return true -- compare instance class and class definition
            end
            -- more thorough tree check
            local classType = nil
            if is_class(value) and is_class(typ) then -- compare class definition and class definition
                classType = value
            else
                classType = getmetatable(value) -- compare instance/userdata and class definition
            end
            while classType ~= nil do
                if classType.class_cast ~= nil and classType.class_cast == typ.class_cast then
                    return true
                elseif classType.__type ~= nil and classType.__type == typ.__type then
                    return true
                elseif classType.__instanceDict ~= nil and classType.__instanceDict == typ.__instanceDict then
                    return true -- maybe deprecated / renamed to __type 
                end
                classType = classType.super
            end
            return false
        end
        return t == typ
    end
    local typeMetaTable = {__call = function(tbl,...) return typeFunc(...) end}
    setmetatable(typeTable, typeMetaTable)
    type = typeTable
end

MathExtensions = extensions.extension()
function MathExtensions:setup()
    math.round = function(value)
        local v = math.floor(value)
        local d = value - v
        if d < 0.5 then
            return v
        end
        return math.ceil(value)
    end
    math.sign = function(value, sign)
        if (sign < 0 and value > 0) or (sign > 0 and value < 0) then
            return -value
        end
        return value
    end
end

QuatExtensions = extensions.extension()
function QuatExtensions:setup()
    function quat_rotate_vec3(q,v)
        local u = vec3(q.x, q.y, q.z)
        local s = q.w
        return 2.0 * u:dot(v) * u + (s*s - u:dot(u)) * v + 2.0 * s * u:cross(v)
    end
    function quat_to_mat3(q)
        x,y,z,w = q.x,q.y,q.z,q.w
        xx, yy, zz = x * x, y * y, z * z
        xy, xz, yz = x * y, x * z, y * z
        sx, sy, sz = x * w, y * w, z * w
        xx2, yy2, zz2 = 2*xx, 2*yy, 2*zz
        xy2, xz2, yz2 = 2*xy, 2*xz, 2*yz
        sx2, sy2, sz2 = 2*sx, 2*sy, 2*sz
        return mat3(
            1 - yy2 - zz2, xy2 - xz2, xz2 + sy2,
            xy2 + sz2, 1 - xx2 - zz2, yz2 + sx2,
            xz2 - sy2, yz2 + sx2, 1 - xx2 - yy2
        )
    end
    function quat_to_mat4(q)
        m = quat_to_mat3(q)
        return mat4(
            m[1][1], m[1][2],m[1][3], 0,
            m[2][1], m[2][2],m[2][3], 0,
            m[3][1], m[3][2],m[3][3], 0,
            0,0,0,1
        )
    end
    function mat_to_quat(m)
        m00, m11, m22 = m[1][1], m[2][2], m[3][3]
        m01, m02, m10, m12, m20, m21 = m[1][2], m[1][3], m[2][1], m[2][3], m[3][1], m[3][2]
        x = math.sqrt( math.max( 0, 1 + m00 - m11 - m22 ) ) / 2;
        y = math.sqrt( math.max( 0, 1 - m00 + m11 - m22 ) ) / 2;
        z = math.sqrt( math.max( 0, 1 - m00 - m11 + m22 ) ) / 2;
        w = math.sqrt( math.max( 0, 1 + m00 + m11 + m22 ) ) / 2;
        return quat(math.sign(x,m21-m12),math.sign(y, m02-m20),math.sign(z,m01-m10),w)
    end
    qt = getmetatable(quat())
    qt.__mul = extensions.hook(qt.__mul, function(base, q, v)
        if type.is(v, vec3) then
            return v * q -- rotate a vec3 (optional)
        end
        if type.is(v, mat3) or type.is(v, mat4) then
            v = v:quat()
        end
        return base(q,v)
    end)
    quat.mat3 = quat_to_mat3
    quat.mat4 = quat_to_mat4
    v3 = getmetatable(vec3())
    v3.__mul = extensions.hook(v3.__mul, function(base, x, y)
        if type.is(y, quat) then
            return quat_rotate_vec3(y, x) -- rotate vec3 with quat
        elseif type.is(y, mat3) or type.is(y, mat4) then
            return y * x -- multiply vec with matrix (optional)
        end
        return base(x,y)
    end)
    m3 = getmetatable(mat3())
    m3.__mul = extensions.hook(m3.__mul, function(base, x, y)
        if type.is(y, quat) then
            y = y:mat3()
        end
        return base(x,y)
    end)
    mat3.quat = mat_to_quat
    m4 = getmetatable(mat4())
    m4.__mul = extensions.hook(m4.__mul, function(base, x, y)
        if type.is(y, vec3) then
            v = base(x,vec4(y.x,y.y,y.z,1))
            return vec3(v.x,v.y,v.z)
        elseif type.is(y, quat) then
            y = y:mat4()
        end
        return base(x,y)
    end)
    mat4.quat = mat_to_quat
end