VTK: Cut Model By Surface And Increase Density

Here are two models tooth and attachment that intersect with each other.
I want to find the outside part of attachment.

I tried vtkBooleanOperationPolyDataFilter and vtkPolyDataBooleanFilter to find the outside part, but it may fail when there is a big triangle cell on the attachment.
Increase density of the model will have a better result. But it’s not safe enoughly if we scale the model. (vtkLinearSubdivisionFilter will not change the shape of model as vtkButterflySubdivisionFilter)

void IncreaseDensity(vtkSPtr<vtkPolyData> pld, const double maxEdgeLen)
{
    vtkSPtrNew( tmpData, vtkPolyData );
    tmpData->DeepCopy( pld );

    // fix issue: Dataset is non-manifold and cannot be subdivided.
    vtkSPtrNew( connectivityFilter, vtkPolyDataConnectivityFilter );
    connectivityFilter->SetInputData( tmpData );
    connectivityFilter->SetExtractionModeToLargestRegion();
    connectivityFilter->Update();

    tmpData = connectivityFilter->GetOutput();

    vtkSPtrNew( triangleFilter, vtkTriangleFilter );
    triangleFilter->SetInputData( tmpData );
    triangleFilter->PassLinesOff();
    triangleFilter->PassVertsOff();
    triangleFilter->Update();


    pld->DeepCopy( triangleFilter->GetOutput() );
    if( pld->NeedToBuildCells() )
    {
        pld->BuildCells();
    }
    pld->BuildLinks();
    // Finish: Dataset is non-manifold and cannot be subdivided.

    auto JudgeNeedHandle = [](vtkPolyData *pld, const double maxEdgeLen) -> bool{
        bool toHandle = false;
        for( int i = 0; i < pld->GetNumberOfCells(); ++i )
        {
            vtkCell *cell = pld->GetCell( i );
            vtkPoints *points = cell->GetPoints();
            for( int j = 0; j < points->GetNumberOfPoints(); ++j )
            {
                PointStruct pt0( points->GetPoint( j ) );
                PointStruct pt1( points->GetPoint( (j+1)%points->GetNumberOfPoints() ) );
                if( (pt0-pt1).Length() > maxEdgeLen )
                {
                    toHandle = true;
                    break;
                }
            }
        }
        return toHandle;
    };


    int times = 0;
    while( JudgeNeedHandle( pld, maxEdgeLen ) )
    {
        vtkSPtrNew( data, vtkPolyData );
        data->DeepCopy( pld );

        vtkSPtrNew( subdivisionFilter, vtkLinearSubdivisionFilter );
        subdivisionFilter->SetGlobalWarningDisplay( 1 );
        subdivisionFilter->SetInputData( data );
        subdivisionFilter->SetNumberOfSubdivisions( 1 );
        subdivisionFilter->Update();

        pld->DeepCopy( subdivisionFilter->GetOutput() );
        times++;
    }
    cout << "times: " << times << ", " << pld->GetNumberOfCells() << endl;
}

Finally, we choose vtkClipPolyData to cut the inside part of attachment, set vtkImplicitPolyDataDistance which computes the signed distance from a point x to the nearest point p on an input vtkPolyData as the cut function.

    // ================== clip ==================
    vtkSPtrNew( distanceFilter, vtkImplicitPolyDataDistance );
    distanceFilter->SetInput( toothReader->GetOutput() );
    distanceFilter->Modified();

    vtkSPtrNew( clipper, vtkClipPolyData );
    clipper->SetInputData( attachReader->GetOutput() );
    clipper->SetClipFunction( distanceFilter );
    clipper->GenerateClippedOutputOn();
    clipper->Update();

    cout << "GetNumberOfCells: " << clipper->GetOutput()->GetNumberOfCells() << endl;

    vtkSPtrNew( finalMapper, vtkPolyDataMapper );
    finalMapper->SetInputData( clipper->GetOutput() );

    vtkSPtrNew( finalActor, vtkActor );
    finalActor->SetMapper( finalMapper );

    renderer->AddActor( finalActor );

But the result of the cutting is not particularly idealistic.
So we have to increase density necessarily.
The left one is the low density model, the right attachment model has higher density in the following image.

All code snippet:

#include <iostream>
#include <vtkSmartPointer.h>
#include <vtkSphereSource.h>
#include <vtkActor.h>
#include <vtkConeSource.h>
#include <vtkRenderer.h>
#include <vtkRenderWindow.h>
#include <vtkPolyDataMapper.h>
#include <vtkRenderWindowInteractor.h>

#include <vtkImplicitPolyDataDistance.h>
#include <vtkProperty.h>
#include <vtkSelectEnclosedPoints.h>
#include <vtkTriangleFilter.h>
#include <vtkSTLReader.h>
#include <vtkPointData.h>
#include <vtkColorTransferFunction.h>
#include <vtkPolyDataConnectivityFilter.h>
#include <vtkLinearSubdivisionFilter.h>
#include <vtkButterflySubdivisionFilter.h>
#include <vtkNamedColors.h>
#include <vtkClipPolyData.h>

#include "point.hpp"

#define vtkSPtr vtkSmartPointer
#define vtkSPtrNew(Var, Type) vtkSPtr<Type> Var = vtkSPtr<Type>::New();

using namespace std;


int main()
{
    vtkSPtrNew( attachReader, vtkSTLReader );
    attachReader->SetFileName( "/Users/weiyang/Desktop/attachment.stl" );
    attachReader->Update();

    vtkSPtrNew( toothReader, vtkSTLReader );
    toothReader->SetFileName( "/Users/weiyang/Desktop/tooth.stl" );
    toothReader->Update();

    vtkSPtrNew( toothMapper, vtkPolyDataMapper );
    toothMapper->SetInputData( toothReader->GetOutput() );

    vtkSPtrNew( toothActor, vtkActor );
    toothActor->SetMapper( toothMapper );
    toothActor->GetProperty()->SetRepresentationToWireframe();

    vtkSPtrNew( renderer, vtkRenderer );
    renderer->AddActor( toothActor );
    renderer->SetBackground( 0, 0, 0 ); //1, 1, 1 );

    // ================== clip ==================
    vtkSPtrNew( distanceFilter, vtkImplicitPolyDataDistance );
    distanceFilter->SetInput( toothReader->GetOutput() );
    distanceFilter->Modified();

    vtkSPtrNew( clipper, vtkClipPolyData );
    clipper->SetInputData( attachReader->GetOutput() );
    clipper->SetClipFunction( distanceFilter );
    clipper->GenerateClippedOutputOn();
    clipper->Update();

    cout << "GetNumberOfCells: " << clipper->GetOutput()->GetNumberOfCells() << endl;

    vtkSPtrNew( finalMapper, vtkPolyDataMapper );
    finalMapper->SetInputData( clipper->GetOutput() );

    vtkSPtrNew( finalActor, vtkActor );
    finalActor->SetMapper( finalMapper );
    finalActor->GetProperty()->SetColor( 1, 0, 0 );

    renderer->AddActor( finalActor );
    // ================== finish: clip ==================

    vtkSPtrNew( renderWindow, vtkRenderWindow );
    renderWindow->AddRenderer( renderer );

    vtkSPtrNew( renderWindowInteractor, vtkRenderWindowInteractor );
    renderWindowInteractor->SetRenderWindow( renderWindow );


    renderer->ResetCamera();
    renderWindow->Render();

    renderWindowInteractor->Start();
    return 0;
}

The STL files can be downloaded:
cutModel_tooth.stl.
cutModel_attachment.stl