OK I got it working:
I forgot to reference the RevitAPIIFC.dll, and the vector needed to be in a StrongBox container in rvt 2017
import clr
clr.AddReference('RevitAPI')
from Autodesk.Revit.DB import *
clr.AddReference('RevitAPIIFC')
from Autodesk.Revit.DB.IFC import *
clr.AddReference('RevitNodes')
import Revit
clr.ImportExtensions(Revit.GeometryConversion)
clr.ImportExtensions(Revit.Elements)
clr.AddReference('RevitServices')
import RevitServices
from RevitServices.Persistence import DocumentManager
from RevitServices.Transactions import TransactionManager
doc = DocumentManager.Instance.CurrentDBDocument
#Preparing input from dynamo to revit
wall = UnwrapElement(IN[0])
familyInst = UnwrapElement(IN[1])
vec = clr.StrongBox[XYZ](wall.Orientation)
#Get curves
curves=ExporterIFCUtils.GetInstanceCutoutFromWall(doc,wall,familyInst,vec)
#Convert to dybmo geometry:
OUT = [curve.ToProtoType(True) for curve in curves]